#pragma once #include "CoreMinimal.h" #include "EngineUtils.h" #include "Runtime/Engine/Public/SceneViewExtension.h" #include "Runtime/Engine/Classes/Engine/Texture2D.h" #include "MaterialShared.h" #include "Materials/MaterialInstance.h" #include "Delegates/Delegate.h" #include "OptiXContext.h" #include "OptiXObjectComponent.h" #include "OptiXLaserComponent.h" #include "OptiXLaserActor.h" #include "OptiXCameraActor.h" // Let's try some events! DECLARE_EVENT(FOptiXContextManager, FLaserTraceFinishedEvent) DECLARE_EVENT_OneParam(FOptiXContextManager, FWavelengthChangedEvent, const float) DECLARE_MULTICAST_DELEGATE(FOnSceneChangedDelegate); // DX #if PLATFORM_WINDOWS #include "AllowWindowsPlatformTypes.h" #endif #include <d3d11.h> #if PLATFORM_WINDOWS #include "HideWindowsPlatformTypes.h" #endif // print cuda error helper: inline void PrintLastCudaError(FString Msg) { cudaError_t Err = cudaGetLastError(); if (cudaSuccess != Err) { UE_LOG(LogTemp, Fatal, TEXT("Cuda Error: %s. "), *Msg, static_cast<int>(Err), cudaGetErrorString(Err)); } } class OPTIX_API FOptiXContextManager : public FSceneViewExtensionBase { public: // The auto register thing is used to make sure this constructor is only called via the NewExtension function FOptiXContextManager(const FAutoRegister& AutoRegister); ~FOptiXContextManager() { if (bIsInitialized) { cudaGraphicsUnregisterResource(CudaResourceDepthLeft); cudaGraphicsUnregisterResource(CudaResourceDepthRight); cudaGraphicsUnregisterResource(CudaResourceColorLeft); cudaGraphicsUnregisterResource(CudaResourceColorRight); cudaGraphicsUnregisterResource(CudaResourceIntersections); cudaGraphicsUnregisterResource(CudaResourceColorOrtho); cudaGraphicsUnregisterResource(CudaResourceDepthOrtho); PrintLastCudaError("cudaGraphicsUnregisterResource"); cudaFree(CudaLinearMemoryDepth); cudaFree(CudaLinearMemoryColor); cudaFree(CudaLinearMemoryIntersections); PrintLastCudaError("cudaFree"); } AccelerationsToDeleteQueue.Empty(); BuffersToDeleteQueue.Empty(); GeometriesToDeleteQueue.Empty(); GeometryGroupToDeleteQueue.Empty(); GeometryInstancesToDeleteQueue.Empty(); GroupsToDeleteQueue.Empty(); MaterialsToDeleteQueue.Empty(); ProgramToDeleteQueue.Empty(); TextureSamplersToDeleteQueue.Empty(); TransformsToDeleteQueue.Empty(); GroupChildrenToRemoveQueue.Empty(); GeometryGroupChildrenToRemoveQueue.Empty(); ComponentsToInitializeQueue.Empty(); ComponentsToUpdateQueue.Empty(); CubemapComponentsToUpdateQueue.Empty(); LaserActor.Reset(); CameraActor.Reset(); } // ISceneViewExtension interface start, called by the render thread: public: virtual void SetupViewFamily(FSceneViewFamily& InViewFamily) override; virtual void SetupView(FSceneViewFamily& InViewFamily, FSceneView& InView) override; virtual void BeginRenderViewFamily(FSceneViewFamily& InViewFamily) override; virtual void PreRenderView_RenderThread(FRHICommandListImmediate& RHICmdList, FSceneView& InView) override; virtual void PreRenderViewFamily_RenderThread(FRHICommandListImmediate& RHICmdList, FSceneViewFamily& InViewFamily) override; virtual void PostRenderViewFamily_RenderThread(FRHICommandListImmediate& RHICmdList, FSceneViewFamily& InViewFamily) override; virtual bool IsActiveThisFrame(class FViewport* InViewport) const override; virtual void PostRenderView_RenderThread(FRHICommandListImmediate& RHICmdList, FSceneView& InView) override; // ISceneViewExtension interface end void RenderOrthoPass(); // Initialization methods, called by the GAME thread void Init(); void EndPlay() { //CleanupOptiXOnEnd(); //bCleanup.AtomicSet(true); //bStartTracing.AtomicSet(false); //bCleanup.AtomicSet(false); //bEndPlay.AtomicSet(true); } UOptiXContext* GetOptiXContext() { return OptiXContext.Get(); } UMaterialInstanceDynamic* GetOptiXMID() // Used to set up the post process { return DynamicMaterial.Get(); } UMaterialInstanceDynamic* GetOptiXMIDOrtho() // Used to set up the post process { return DynamicMaterialOrtho.Get(); } void SceneChangedCallback(); // The OptiX context is not thread-safe, so any changes to variables/properties on the game thread // while a trace is running will lead to errors. Game thread objects push their requested updates into several queues, // which then get updated just before the trace runs on the render thread. // TQueue is guaranteed to be thread-safe. // There is probably a better and faster way of doing this. void RegisterOptiXComponent(IOptiXComponentInterface* Component) { ComponentsToInitializeQueue.Enqueue(Component); } void QueueComponentUpdate(IOptiXComponentInterface* Component) { ComponentsToUpdateQueue.Enqueue(Component); } void RequestCubemapUpdate(UOptiXCubemapComponent* Component) { CubemapComponentsToUpdateQueue.Enqueue(Component); } void SetActiveLaserActor(AOptiXLaserActor* Laser) { LaserActor = Laser; if (LaserMaterialDynamic.IsValid()) { LaserActor->SetLaserMaterial(LaserMaterialDynamic.Get()); } else { UE_LOG(LogTemp, Error, TEXT("LaserMaterialDynamic is invalid!")); } } void SetActiveCameraActor(AOptiXPlayerCameraManager* Cam) { CameraActor = Cam; } FIntRect GetViewRectanglePerEye() { return FIntRect(0, 0, Width, Height); } int32 RequestCubemapId(); void DeleteCubemapId(int32 Id); void AddCubemapToBuffer(int32 CubemapId, int32 SamplerId); void BroadcastWavelengthChange(float WL) { WavelengthChangedEvent.Broadcast(WL); //UE_LOG(LogTemp, Error, TEXT("LaserMaterialDynamic is invalid!")); } public: FThreadSafeBool bStartTracing = false; FThreadSafeBool bIsInitialized = false; FThreadSafeBool bLaserIsInitialized = false; FThreadSafeBool bSceneChanged = true; FThreadSafeBool bIsTracing = false; FThreadSafeBool bClearToLaunch = true; FThreadSafeBool bCleanup = false; FThreadSafeBool bValidCubemap = false; FThreadSafeBool bRequestOrthoPass = false; //FThreadSafeBool bEndPlay = false; FLaserTraceFinishedEvent LaserTraceFinishedEvent; FWavelengthChangedEvent WavelengthChangedEvent; FOnSceneChangedDelegate OnSceneChangedDelegate; public: // This is so ugly but the optix api structure doesn't really allow anything more abstract. TQueue<optix::Acceleration> AccelerationsToDeleteQueue; TQueue<optix::Buffer> BuffersToDeleteQueue; TQueue<optix::Geometry> GeometriesToDeleteQueue; TQueue<optix::GeometryGroup> GeometryGroupToDeleteQueue; TQueue<optix::GeometryInstance> GeometryInstancesToDeleteQueue; TQueue<optix::Group> GroupsToDeleteQueue; TQueue<optix::Material> MaterialsToDeleteQueue; TQueue<optix::Program> ProgramToDeleteQueue; TQueue<optix::TextureSampler> TextureSamplersToDeleteQueue; TQueue<optix::Transform> TransformsToDeleteQueue; TQueue<TPair<optix::Group, uint32>> GroupChildrenToRemoveQueue; TQueue<TPair<optix::GeometryGroup, uint32>> GeometryGroupChildrenToRemoveQueue; FMatrix OrthoMatrix; private: void InitContext(); void InitRendering(); void InitBuffers(); void InitPrograms(); void InitLaser(); void LaunchLaser(); void InitCubemap(); void InitOptiXComponents(FRHICommandListImmediate & RHICmdList) { // Possibly dangerous? Use limited for instead of while in case something goes wrong and we deadlock for (uint32 i = 0; i < 100 && !ComponentsToInitializeQueue.IsEmpty(); i++) { IOptiXComponentInterface* Component; if (ComponentsToInitializeQueue.Dequeue(Component)) { Component->InitOptiXComponent(RHICmdList); } } } void UpdateOptiXComponentVariables() { for (uint32 i = 0; i < 100 && !ComponentsToUpdateQueue.IsEmpty(); i++) { IOptiXComponentInterface* Component; if (ComponentsToUpdateQueue.Dequeue(Component)) { if (Component != nullptr) { Component->UpdateOptiXComponentVariables(); Component->SetUpdateQueued(false);// bUpdateQueued.AtomicSet(false); } } } } void RemovePendingChildrenFromGroups() { for (uint32 i = 0; i < 100 && !GroupChildrenToRemoveQueue.IsEmpty(); i++) { TPair<optix::Group, uint32> Pair; if (GroupChildrenToRemoveQueue.Dequeue(Pair)) { if (Pair.Key != NULL) { Pair.Key->removeChild(Pair.Value); // TODO do we need to do anything else here? } else { UE_LOG(LogTemp, Error, TEXT("Error while trying to remove optix child in queue: Object was NULL")); } } } for (uint32 i = 0; i < 100 && !GeometryGroupChildrenToRemoveQueue.IsEmpty(); i++) { TPair<optix::GeometryGroup, uint32> Pair; if (GeometryGroupChildrenToRemoveQueue.Dequeue(Pair)) { if (Pair.Key != NULL) { Pair.Key->removeChild(Pair.Value); // TODO do we need to do anything else here? } else { UE_LOG(LogTemp, Error, TEXT("Error while trying to remove optix child in queue: Object was NULL")); } } } } void DestroyOptiXObjects() { for (uint32 i = 0; i < 100 && !AccelerationsToDeleteQueue.IsEmpty(); i++) { optix::Acceleration NativeObj; if (AccelerationsToDeleteQueue.Dequeue(NativeObj)) { if (NativeObj != NULL) { if(NativeObj->getContext() != NULL) NativeObj->destroy(); // TODO do we need to do anything else here? else { UE_LOG(LogTemp, Warning, TEXT("Context already destroyed but somehow this buffer handle is still valid")); } } else { UE_LOG(LogTemp, Error, TEXT("Error while trying to destroy optix object in queue: Object was NULL")); } } } for (uint32 i = 0; i < 100 && !BuffersToDeleteQueue.IsEmpty(); i++) { optix::Buffer NativeObj; if (BuffersToDeleteQueue.Dequeue(NativeObj)) { if (NativeObj != NULL) { try { NativeObj->destroy(); // TODO do we need to do anything else here? } catch (optix::Exception& E) { FString Message = FString(E.getErrorString().c_str()); UE_LOG(LogTemp, Error, TEXT("Trying to remove buffer: OptiX Error: %s"), *Message); GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, FString::Printf(TEXT("OptiX Error %s"), *Message)); } } else { UE_LOG(LogTemp, Error, TEXT("Error while trying to destroy optix object in queue: Object was NULL")); } } } for (uint32 i = 0; i < 100 && !GeometriesToDeleteQueue.IsEmpty(); i++) { optix::Geometry NativeObj; if (GeometriesToDeleteQueue.Dequeue(NativeObj)) { if (NativeObj != NULL) { NativeObj->destroy(); // TODO do we need to do anything else here? } else { UE_LOG(LogTemp, Error, TEXT("Error while trying to destroy optix object in queue: Object was NULL")); } } } for (uint32 i = 0; i < 100 && !GeometryInstancesToDeleteQueue.IsEmpty(); i++) { optix::GeometryInstance NativeObj; if (GeometryInstancesToDeleteQueue.Dequeue(NativeObj)) { if (NativeObj != NULL) { NativeObj->destroy(); // TODO do we need to do anything else here? } else { UE_LOG(LogTemp, Error, TEXT("Error while trying to destroy optix object in queue: Object was NULL")); } } } for (uint32 i = 0; i < 100 && !GeometryGroupToDeleteQueue.IsEmpty(); i++) { optix::GeometryGroup NativeObj; if (GeometryGroupToDeleteQueue.Dequeue(NativeObj)) { if (NativeObj != NULL) { NativeObj->destroy(); // TODO do we need to do anything else here? } else { UE_LOG(LogTemp, Error, TEXT("Error while trying to destroy optix object in queue: Object was NULL")); } } } for (uint32 i = 0; i < 100 && !GroupsToDeleteQueue.IsEmpty(); i++) { optix::Group NativeObj; if (GroupsToDeleteQueue.Dequeue(NativeObj)) { if (NativeObj != NULL) { NativeObj->destroy(); // TODO do we need to do anything else here? } else { UE_LOG(LogTemp, Error, TEXT("Error while trying to destroy optix object in queue: Object was NULL")); } } } for (uint32 i = 0; i < 100 && !MaterialsToDeleteQueue.IsEmpty(); i++) { optix::Material NativeObj; if (MaterialsToDeleteQueue.Dequeue(NativeObj)) { if (NativeObj != NULL) { NativeObj->destroy(); // TODO do we need to do anything else here? } else { UE_LOG(LogTemp, Error, TEXT("Error while trying to destroy optix object in queue: Object was NULL")); } } } for (uint32 i = 0; i < 100 && !ProgramToDeleteQueue.IsEmpty(); i++) { optix::Program NativeObj; if (ProgramToDeleteQueue.Dequeue(NativeObj)) { if (NativeObj != NULL) { NativeObj->destroy(); // TODO do we need to do anything else here? } else { UE_LOG(LogTemp, Error, TEXT("Error while trying to destroy optix object in queue: Object was NULL")); } } } for (uint32 i = 0; i < 100 && !TextureSamplersToDeleteQueue.IsEmpty(); i++) { optix::TextureSampler NativeObj; if (TextureSamplersToDeleteQueue.Dequeue(NativeObj)) { if (NativeObj != NULL) { try { NativeObj->destroy(); // TODO do we need to do anything else here? } catch (optix::Exception& E) { FString Message = FString(E.getErrorString().c_str()); UE_LOG(LogTemp, Error, TEXT("Trying to remove texture sampler: OptiX Error: %s"), *Message); GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Red, FString::Printf(TEXT("OptiX Error %s"), *Message)); } } else { UE_LOG(LogTemp, Error, TEXT("Error while trying to destroy optix object in queue: Object was NULL")); } } } for (uint32 i = 0; i < 100 && !TransformsToDeleteQueue.IsEmpty(); i++) { optix::Transform NativeObj; if (TransformsToDeleteQueue.Dequeue(NativeObj)) { if (NativeObj != NULL) { NativeObj->destroy(); // TODO do we need to do anything else here? } else { UE_LOG(LogTemp, Error, TEXT("Error while trying to destroy optix object in queue: Object was NULL")); } } } } void UpdateRequestedCubemaps(FRHICommandListImmediate & RHICmdList) { // update only the first for now, shouldn't be more than 1 in queue anyway: if (!CubemapComponentsToUpdateQueue.IsEmpty()) { UOptiXCubemapComponent* Comp; if (CubemapComponentsToUpdateQueue.Dequeue(Comp)) { Comp->UpdateCubemap(RHICmdList); } } } void UpdateCubemapBuffer(FRHICommandListImmediate & RHICmdList); void CleanupOptiXOnEnd() { UE_LOG(LogTemp, Display, TEXT("Starting Cleanup in Context Manager")); if (bIsInitialized) { if(CudaResourceDepthLeft != NULL) cudaGraphicsUnregisterResource(CudaResourceDepthLeft); if (CudaResourceDepthRight != NULL) cudaGraphicsUnregisterResource(CudaResourceDepthRight); if (CudaResourceColorLeft != NULL) cudaGraphicsUnregisterResource(CudaResourceColorLeft); if(CudaResourceColorRight != NULL) cudaGraphicsUnregisterResource(CudaResourceColorRight); if(CudaResourceIntersections != NULL) cudaGraphicsUnregisterResource(CudaResourceIntersections); if (CudaResourceDepthOrtho != NULL) cudaGraphicsUnregisterResource(CudaResourceDepthOrtho); if (CudaResourceColorOrtho != NULL) cudaGraphicsUnregisterResource(CudaResourceColorOrtho); PrintLastCudaError("cudaGraphicsUnregisterResource"); if(CudaLinearMemoryDepth != NULL) cudaFree(CudaLinearMemoryDepth); if (CudaLinearMemoryColor != NULL) cudaFree(CudaLinearMemoryColor); if (CudaLinearMemoryIntersections != NULL) cudaFree(CudaLinearMemoryIntersections); PrintLastCudaError("cudaFree"); } //check(IsInRenderingThread()); bStartTracing.AtomicSet(false); //bEndPlay.AtomicSet(false); bIsInitialized.AtomicSet(false); bLaserIsInitialized.AtomicSet(false); bSceneChanged.AtomicSet(true); bIsTracing.AtomicSet(false); bClearToLaunch.AtomicSet(true); bCleanup.AtomicSet(false); bValidCubemap.AtomicSet(false); // Clear the queue DestroyOptiXObjects(); if (OptiXContext.IsValid()) { OptiXContext->RemoveFromRoot(); } if (LaserOutputBuffer.IsValid()) { LaserOutputBuffer->RemoveFromRoot(); } if (OutputTexture.IsValid()) { OutputTexture->RemoveFromRoot(); } if (OutputTexture2.IsValid()) { OutputTexture2->RemoveFromRoot(); } if (DepthTexture.IsValid()) { DepthTexture->RemoveFromRoot(); } if (DepthTexture2.IsValid()) { DepthTexture2->RemoveFromRoot(); } if (CubemapSampler.IsValid()) { CubemapSampler->RemoveFromRoot(); } if (CubemapBuffer.IsValid()) { CubemapBuffer->RemoveFromRoot(); } OutputBuffer.Reset(); OutputDepthBuffer.Reset(); OutputTexture.Reset(); DepthTexture.Reset(); OutputTexture2.Reset(); DepthTexture2.Reset(); LaserIntersectionTexture.Reset(); DynamicMaterial.Reset(); DynamicMaterialOrtho.Reset(); RegularMaterial.Reset(); VRMaterial.Reset(); LaserMaterial.Reset(); LaserMaterialDynamic.Reset(); LaserActor.Reset(); TopObject.Reset(); TopAcceleration.Reset(); OptiXContext.Reset(); if (NativeContext != NULL) { NativeContext->destroy(); NativeContext = NULL; } /*{ OptiXContext->RemoveFromRoot(); LaserOutputBuffer->RemoveFromRoot(); OutputTexture->RemoveFromRoot(); OutputTexture2->RemoveFromRoot(); DepthTexture->RemoveFromRoot(); DepthTexture2->RemoveFromRoot(); OutputBuffer.Reset(); OutputDepthBuffer.Reset(); OutputTexture.Reset(); DepthTexture.Reset(); OutputTexture2.Reset(); DepthTexture2.Reset(); DynamicMaterial.Reset(); RegularMaterial.Reset(); VRMaterial.Reset(); TopObject.Reset(); TopAcceleration.Reset(); OptiXContext.Reset(); if (NativeContext != NULL) { NativeContext->destroy(); } }*/ } private: // OptiX Part // Todo: refactor this to delegates maybe? TQueue<IOptiXComponentInterface*> ComponentsToInitializeQueue; TQueue<IOptiXComponentInterface*> ComponentsToUpdateQueue; TQueue<UOptiXCubemapComponent*> CubemapComponentsToUpdateQueue; TWeakObjectPtr<AOptiXLaserActor> LaserActor; TWeakObjectPtr<AOptiXPlayerCameraManager> CameraActor; // OptiX Objects to be kept in the context manager, TODO triple check that the GC doesn't nab them. // Those are always required, but some should also be changeable! TODO make custom setters for them later. TWeakObjectPtr<UOptiXContext> OptiXContext; TWeakObjectPtr<UOptiXProgram> RayGenerationProgram; TWeakObjectPtr<UOptiXProgram> MissProgram; TWeakObjectPtr<UOptiXProgram> ExceptionProgram; TWeakObjectPtr<UOptiXGroup> TopObject; TWeakObjectPtr<UOptiXAcceleration> TopAcceleration; TWeakObjectPtr<UOptiXBuffer> OutputBuffer; TWeakObjectPtr<UOptiXBuffer> OutputDepthBuffer; TWeakObjectPtr<UOptiXTextureSampler> CubemapSampler; TWeakObjectPtr<UOptiXBuffer> CubemapBuffer; TWeakObjectPtr<UOptiXBuffer> CubemapsInputBuffer; TQueue<int32> UnallocatedCubemapIds; TArray<TArray<FColor>> SurfaceDataCube; optix::Context NativeContext; // Laser Part TWeakObjectPtr<UOptiXBuffer> LaserOutputBuffer; TWeakObjectPtr<UOptiXProgram> LaserRayGenerationProgram; TWeakObjectPtr<UOptiXProgram> LaserMissProgram; TWeakObjectPtr<UOptiXProgram> LaserExceptionProgram; // Rendering Part //FUpdateTextureRegion2D TextureRegion; //UTextureCube* TextureCube; int32 Width; int32 Height; FTexture2DRHIRef OutputTextureColorRightRef; FTexture2DRHIRef OutputTextureColorLeftRef; FTexture2DRHIRef OutputTextureDepthRightRef; FTexture2DRHIRef OutputTextureDepthLeftRef; FTexture2DRHIRef OutputTextureColorOrthoRef; FTexture2DRHIRef OutputTextureDepthOrthoRef; TWeakObjectPtr<UTexture2D> OutputTexture; TWeakObjectPtr<UTexture2D> DepthTexture; TWeakObjectPtr<UTexture2D> OutputTexture2; TWeakObjectPtr<UTexture2D> DepthTexture2; TWeakObjectPtr<UTexture2D> OutputTextureOrtho; TWeakObjectPtr<UTexture2D> DepthTextureOrtho; TWeakObjectPtr<UMaterialInstanceDynamic> DynamicMaterial; TWeakObjectPtr<UMaterialInstanceDynamic> DynamicMaterialOrtho; TWeakObjectPtr<UMaterial> RegularMaterial; TWeakObjectPtr<UMaterial> VRMaterial; bool bWithHMD; // --------------------------------------------------------------------------------------------------- // Laser stuff // --------------------------------------------------------------------------------------------------- FTexture2DRHIRef LaserIntersectionTextureRef; TWeakObjectPtr<UTexture2D> LaserIntersectionTexture; TWeakObjectPtr<UMaterial> LaserMaterial; TWeakObjectPtr<UMaterialInstanceDynamic> LaserMaterialDynamic; public: TQueue<TPair<uint32, TArray<FVector>>> LaserIntersectionQueue; TArray<TArray<FVector>> PreviousLaserResults; private: TArray<FVector4> IntersectionData; TArray<FVector4> OldIntersectionData; FThreadSafeBool bTracingLaser; FCriticalSection CriticalSection; int32 LaserMaxDepth; int32 LaserEntryPoint; int32 LaserBufferSize; int32 LaserBufferWidth; int32 LaserBufferHeight; // --------------------------------------------------------------------------------------------------- // DX <-> CUDA stuff // --------------------------------------------------------------------------------------------------- public: // I'm not sure which ones I actually need IDXGIAdapter *CudaCapableAdapter = NULL; // Adapter to use ID3D11Device *D3DDevice = NULL; // Rendering device ID3D11DeviceContext *D3DDeviceContext = NULL; // RTX mode int RTXOn = 1; private: void InitCUDADX(); //ID3D11Texture2D* D3D11Texture; cudaGraphicsResource* CudaResourceDepthLeft; cudaGraphicsResource* CudaResourceDepthRight; cudaGraphicsResource* CudaResourceColorLeft; cudaGraphicsResource* CudaResourceColorRight; cudaGraphicsResource* CudaResourceIntersections; cudaGraphicsResource* CudaResourceColorOrtho; cudaGraphicsResource* CudaResourceDepthOrtho; void* CudaLinearMemoryDepth; void* CudaLinearMemoryColor; void* CudaLinearMemoryIntersections; size_t Pitch; // fix me size_t PitchLaser; cudaGraphicsResource *Resources[7]; };