how can I add denoiser in OWL?

[StringHeart]

I try to copy optix7course's ex11_denoiseColorOnly in owl.

And I use DEVICE DeviceMemory in DeviceMemory.h to replace CUDABuffer in optix7course. I use OWLViewer::fbPointer as the inputLayer.data so that the format is OPTIX_PIXEL_FORMAT_UCHAR4.

I thought it will run normally. but I get an error code 7800 which is 'OPTIX_ERROR_NOT_SUPPORTED'. I can run ex11 successfully so I wasn't my device's problem. It must be some mistake I make.

Variable declarations:

    DeviceMemory         denoisedBuffer;
    OptixDenoiser        myDenoiser = nullptr;
    DeviceMemory         denoiserScratch;
    DeviceMemory         denoiserState;

code in render and resize:

    // render
    OptixDenoiserParams denoiserParams;
    denoiserParams.denoiseAlpha = 1;
    denoiserParams.hdrIntensity = (CUdeviceptr)0;
    denoiserParams.blendFactor = 1.0f / (frameID + 1);
    // -------------------------- input ---------------------------
    OptixImage2D inputLayer;
    inputLayer.data = (CUdeviceptr)fbPointer;
    inputLayer.width = fbSize.x;
    inputLayer.height = fbSize.y;
    inputLayer.rowStrideInBytes = fbSize.x * sizeof(uint32_t);
    inputLayer.pixelStrideInBytes = sizeof(uint32_t);
    inputLayer.format = OPTIX_PIXEL_FORMAT_UCHAR4;  // RGBA
    // -------------------------- output --------------------------
    OptixImage2D outputLayer;
    outputLayer.data = (CUdeviceptr)denoisedBuffer.get();
    outputLayer.width = fbSize.x;
    outputLayer.height = fbSize.y;
    outputLayer.rowStrideInBytes = fbSize.x * sizeof(uint32_t);
    outputLayer.pixelStrideInBytes = sizeof(uint32_t);
    outputLayer.format = OPTIX_PIXEL_FORMAT_UCHAR4;

    // --------------------------------------------------------------------
    if (true) {
    #if OPTIX_VERSION >= 70300
      OptixDenoiserGuideLayer denoiserGuideLayer = {};

      OptixDenoiserLayer denoiserLayer = {};
      denoiserLayer.input = inputLayer;
      denoiserLayer.output = outputLayer;

      OPTIX_CHECK(optixDenoiserInvoke(myDenoiser,
                                /*stream*/0,
                                &denoiserParams,
                                (CUdeviceptr)denoiserState.get(),
                                denoiserState.size(),
                                &denoiserGuideLayer,
                                &denoiserLayer,1,
                                /*inputOffsetX*/0,
                                /*inputOffsetY*/0,
                                (CUdeviceptr)denoiserScratch.get(),
                                denoiserScratch.size()));
    #else
      OPTIX_CHECK(optixDenoiserInvoke(myDenoiser,
                                /*stream*/0,
                                &denoiserParams,
                                (CUdeviceptr)denoiserState.get(),
                                denoiserState.size(),
                                &inputLayer,1,
                                /*inputOffsetX*/0,
                                /*inputOffsetY*/0,
                                &outputLayer,
                                (CUdeviceptr)denoiserScratch.get(),
                                denoiserScratch.size()));
    #endif
    } else {
      cudaMemcpy((void*)outputLayer.data,(void*)inputLayer.data,
                 outputLayer.width*outputLayer.height*sizeof(uint32_t),
                 cudaMemcpyDeviceToDevice);
    }
    cudaDeviceSynchronize(); 

and resize:

    LOG("RESIZE!")
    if (fbColor) {
      owlBufferDestroy(fbColor);
    }

    if (myDenoiser) {
      OPTIX_CHECK(optixDenoiserDestroy(myDenoiser));
    }

    // ---------------------------------------------------------------------------------
    OptixDeviceContext optixContext = (OptixDeviceContext)owlContextGetOptixContext(context, 0);
    OptixDenoiserOptions denoiserOptions = {};
    #if OPTIX_VERSION >= 70300
      OPTIX_CHECK(optixDenoiserCreate(optixContext, OPTIX_DENOISER_MODEL_KIND_LDR,
                    &denoiserOptions,&myDenoiser));
    #else
      denoiserOptions.inputKind = OPTIX_DENOISER_INPUT_RGB;

    #if OPTIX_VERSION < 70100
    // these only exist in 7.0, not 7.1
      denoiserOptions.pixelFormat = OPTIX_PIXEL_FORMAT_UCHAR4;
    #endif

      OPTIX_CHECK(optixDenoiserCreate(optixContext,&denoiserOptions,&myDenoiser));
      OPTIX_CHECK(optixDenoiserSetModel(myDenoiser,OPTIX_DENOISER_MODEL_KIND_LDR,NULL,0));
    #endif

    // .. then compute and allocate memory resources for the myDenoiser
    OptixDenoiserSizes denoiserReturnSizes;
    OPTIX_CHECK(optixDenoiserComputeMemoryResources(myDenoiser,newSize.x,newSize.y,
                                                    &denoiserReturnSizes));

    #if OPTIX_VERSION < 70100
      denoiserScratch.alloc(denoiserReturnSizes.recommendedScratchSizeInBytes);
    #else
      denoiserScratch.alloc(max(denoiserReturnSizes.withOverlapScratchSizeInBytes,
                            denoiserReturnSizes.withoutOverlapScratchSizeInBytes));
    #endif
      denoiserState.alloc(denoiserReturnSizes.stateSizeInBytes);

    this->fbSize = newSize;

    // ------------------------------------------------------------------------------------
    // resize denoisedBuffer
    denoisedBuffer.alloc(fbSize.x * fbSize.y * sizeof(uint32_t));

    fbColor = owlDeviceBufferCreate(context, OWL_FLOAT4, fbSize.x * fbSize.y, nullptr);

    fbRadianceAccum = owlHostPinnedBufferCreate(context, OWL_FLOAT, fbSize.x * fbSize.y);

    owlParamsSetBuffer(launchParams, "frame.fbColor", fbColor);
    owlParamsSetBuffer(launchParams, "fbRadianceAccum", fbRadianceAccum);
    owlParamsSet1ul(launchParams, "frame.fbFinal", (uint64_t)fbPointer);
    owlParamsSet2i(launchParams, "frame.fbSize", (const owl2i&)fbSize);

    // and re-set the camera, since aspect may have changed
    setCamera(lastSetCamera);

    // -----------------------------------------------------------------------------------
    OPTIX_CHECK(optixDenoiserSetup(myDenoiser,0,
                                   newSize.x,newSize.y,
                                   (CUdeviceptr)denoiserState.get(),
                                   denoiserState.size(),
                                   (CUdeviceptr)denoiserScratch.get(),
                                   denoiserScratch.size()));
  }

I want to know where my mistake is.....

[ingowald]

Huh. Interesting question. No obvious answer to me at least.

One thing that comes to mind is that maybe pixel format uchar4 isn't supported with the rest of the config - one simple test would be to take the optix7course and change that to pixel format uchar4 and see if that throws the same error. The output would then be garbage of course if the pixels are really float4 in that sample - but at least it'd be interesting to see if that still renders (garbage) instead of throwing this error.

The only other thing (and hard to judge w/o full source code) is that in the snippets above you say "in resize" and then have a denoiserdestroy in there. you sure you want to destroy the denoiser when you resize? I think you'd only want to resize and reconfig.

Third possibility is that some of the buffers in that sample might be host pinned, and the denoiser might only work on device buffers. Note I'm not saying that it will work only with those; just saying that that's something I'd look at with nothing else to go from.

If you do have an accessible version of that somewhere it might be easier to look at the full code.

[StringHeart]

thank you very much!
I resolve this problem. It's indeed does not support pixel format uchar4 in my computer with Optix version 7.2, and the input buffer is in managed memory. So perhaps both of the problems will lead to the error code.

[ingowald]

Nice; probably the pixel format the main problem then.
If you ever do create a sample that does use the denoiser (ideally as a separate sample that makes clear that this sample is about denoising) then I'd be more than happy to accept a PR with that! :-)