//------------------------------------------------------------------------------ // Project Phoenix // // Copyright (c) 2017-2018 RWTH Aachen University, Germany, // Virtual Reality & Immersive Visualization Group. //------------------------------------------------------------------------------ // License // // Licensed under the 3-Clause BSD License (the "License"); // you may not use this file except in compliance with the License. // See the file LICENSE for the full text. // You may obtain a copy of the License at // // https://opensource.org/licenses/BSD-3-Clause // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. //------------------------------------------------------------------------------ #ifndef TESTS_TEST_UTILITIES_OPENGL_BUFFER_DATA_COMPARISON_HPP_ #define TESTS_TEST_UTILITIES_OPENGL_BUFFER_DATA_COMPARISON_HPP_ #include <cassert> #include <cstdio> #include <iostream> #include <limits> #include <memory> #include <random> #include <sstream> #include <stdexcept> #include <string> #include <tuple> #include <utility> #include "boost/filesystem/operations.hpp" #include "boost/filesystem/path.hpp" #include "catch/catch.hpp" #include "phx/core/logger.hpp" #include "phx/rendering/backend/opengl_image_buffer_data.hpp" #include "phx/resources/resource_manager.hpp" #include "phx/resources/types/image.hpp" #include "test_utilities/reference_image_path.hpp" namespace test_utilities { class OpenGLBufferComparison { public: template <typename T> static double Similarity(const phx::OpenGLImageBufferData<T>& buffer1, const phx::OpenGLImageBufferData<T>& buffer2); // Compare the two image buffers, and if they are not similar enough, // write both to disk and name their filenames and similarity template <typename T> static void REQUIRE_SIMILARITY( const phx::OpenGLImageBufferData<T>& buffer_test, const phx::OpenGLImageBufferData<T>& buffer_reference, double minimumSimilarity = 1.0); template <typename T> static void REQUIRE_REFERENCE_IMAGE_SIMILARITY( const phx::OpenGLImageBufferData<T>& buffer_test, const std::string& filename_reference_image, double minimumSimilarity = 1.0, unsigned int bit_format = 32); private: template <typename T> static double SimilarityAreaNormalized( const phx::OpenGLImageBufferData<T>& buffer1, const phx::OpenGLImageBufferData<T>& buffer2); template <typename T> static double ComputeSimilarity( const phx::OpenGLImageBufferData<T>& buffer_test, phx::Image* ref_image); template <typename T> static double ComputeTotalSimilarity( const phx::OpenGLImageBufferData<T>& buffer1, const phx::OpenGLImageBufferData<T>& buffer2); template <typename T> static double ComputeTotalSimilarity( const phx::OpenGLImageBufferData<T>& buffer1, const phx::OpenGLImageBufferData<T>& buffer2, double max_distance); template <typename T> static double ComputePixelSimilarity( const phx::OpenGLImageBufferData<T>& buffer1, const phx::OpenGLImageBufferData<T>& buffer2, std::size_t x, std::size_t y, double max_distance); template <typename T> static bool DifferentSizes(const T& buffer1, const T& buffer2) { return buffer1.GetWidth() != buffer2.GetWidth() || buffer1.GetHeight() != buffer2.GetHeight(); } static std::string GenerateRandomString(std::size_t length); // @TODO - Does this belong here? Is this something an phx::Image should offer // out of the box? static std::string GetFormatString(phx::Image* image); template <typename T> static std::tuple<std::string, std::string, std::string> SaveTemporaryFiles( const phx::OpenGLImageBufferData<T>& buffer_test, phx::Image* ref_image, phx::OpenGLImageBufferData<phx::OpenGLImageBufferDataType_Float32>* buffer_diff); template <typename T> static void OutputFailMessage( const phx::OpenGLImageBufferData<T>& buffer_test, const std::string& filename_reference_image, double similarity, phx::Image* ref_image); static void OutputComparisonInfo( double minimumSimilarity, double similarity, phx::Image* ref_image, phx::OpenGLImageBufferData<phx::OpenGLImageBufferDataType_Float32>* diff_image, const std::string& file1, const std::string& file2, const std::string& diff_image_name); static void OutputRefImageMissingInfo(const std::string& file1); static std::string ReadInput(); template <typename T> static void DetermineTestOutcome( double similarity, double minimumSimilarity, const phx::OpenGLImageBufferData<T>& buffer_test, const std::string& filename_reference_image, const std::string& temp_file1, const std::string& temp_file2, const std::string& temp_file_diff, bool ref_image_exists); static void OutputUpdateRefImageInfo( const std::string& filename_reference_image, bool overwrite); static void RemoveTempFiles(const std::string& temp_file1, const std::string& temp_file2, const std::string& temp_file_diff); }; template <typename T> double OpenGLBufferComparison::Similarity( const phx::OpenGLImageBufferData<T>& buffer1, const phx::OpenGLImageBufferData<T>& buffer2) { if (DifferentSizes(buffer1, buffer2)) { return 0.0; } return SimilarityAreaNormalized(buffer1, buffer2); } template <typename T> double OpenGLBufferComparison::SimilarityAreaNormalized( const phx::OpenGLImageBufferData<T>& buffer1, const phx::OpenGLImageBufferData<T>& buffer2) { const double total_similarity = ComputeTotalSimilarity(buffer1, buffer2); const double buffer_area = static_cast<double>(buffer1.GetArea()); return total_similarity / buffer_area; } template <typename T> void OpenGLBufferComparison::REQUIRE_SIMILARITY( const phx::OpenGLImageBufferData<T>& buffer_test, const phx::OpenGLImageBufferData<T>& buffer_reference, double minimumSimilarity /*= 1.0*/) { // compute similarity double similarity = Similarity(buffer_reference, buffer_test); // not similar enough? if (similarity < minimumSimilarity) { // create temporary file names std::string randomstring = GenerateRandomString(10); std::string file1 = "buffer_comparison_" + randomstring + "_test.png"; std::string file2 = "buffer_comparison_" + randomstring + "_reference.png"; std::string file_diff = "buffer_comparison_" + randomstring + "_diffmag.png"; // compute difference image buffer auto buffer_diffmag = phx::OpenGLImageBufferData<T>::CreateDifferenceMagnitudeBuffer( buffer_test, buffer_reference); REQUIRE(buffer_diffmag != nullptr); // write files there buffer_test.SaveToFilePNG(file1); buffer_reference.SaveToFilePNG(file2); buffer_diffmag->SaveToFilePNG(file_diff); // output info std::stringstream ss; ss << "Similarity between image buffers should be at least " << minimumSimilarity << ", but is only " << similarity << ".\nImage buffers written to \n" << file1 << " and\n" << file2 << ", and the difference magnitude image to\n" << file_diff; INFO(ss.str()); // fail REQUIRE(similarity >= minimumSimilarity); } else { REQUIRE(similarity >= minimumSimilarity); } } template <typename T> void OpenGLBufferComparison::REQUIRE_REFERENCE_IMAGE_SIMILARITY( const phx::OpenGLImageBufferData<T>& buffer_test, const std::string& filename_reference_image, double minimumSimilarity /*= 1.0*/, unsigned int bit_format /*= 32*/) { double similarity = -std::numeric_limits<double>::infinity(); std::string filename_with_path = test_utilities::reference_image_root + filename_reference_image; auto image = phx::ResourceManager::instance().DeclareResource<phx::Image>( phx::ResourceUtils::DeclarationFromFile( filename_with_path, {{"bit_format", bit_format}}, true)); if (boost::filesystem::exists(boost::filesystem::path(filename_with_path))) { try { image.Load(); } catch (const std::exception&) { phx::error( "Unexpected exception while loading the reference image. File exists " "but may be corrupted."); } } similarity = ComputeSimilarity(buffer_test, image.Get()); if (similarity < minimumSimilarity) { std::unique_ptr< phx::OpenGLImageBufferData<phx::OpenGLImageBufferDataType_Float32>> buffer_diff = nullptr; if (image.Get() != nullptr) { auto buffer_ref = phx::OpenGLImageBufferData<T>::CreateFromImage(image.Get()); buffer_diff = phx::OpenGLImageBufferData<T>::CreateDifferenceMagnitudeBuffer( buffer_test, *buffer_ref.get()); } auto filenames = SaveTemporaryFiles(buffer_test, image.Get(), buffer_diff.get()); std::string file1 = std::get<0>(filenames); std::string file2 = std::get<1>(filenames); std::string file_diff = std::get<2>(filenames); OutputFailMessage(buffer_test, filename_with_path, similarity, image.Get()); if (image.Get() != nullptr) { OutputComparisonInfo(minimumSimilarity, similarity, image.Get(), buffer_diff.get(), file1, file2, file_diff); } else { OutputRefImageMissingInfo(file1); } DetermineTestOutcome(similarity, minimumSimilarity, buffer_test, filename_with_path, file1, file2, file_diff, image.Get() != nullptr); } else { // pass test REQUIRE(similarity >= minimumSimilarity); } } template <typename T> double OpenGLBufferComparison::ComputeSimilarity( const phx::OpenGLImageBufferData<T>& buffer_test, phx::Image* ref_image) { if (ref_image == nullptr) return -std::numeric_limits<double>::infinity(); auto buffer_reference = phx::OpenGLImageBufferData<T>::CreateFromImage(ref_image); if (buffer_reference == nullptr) return -std::numeric_limits<double>::infinity(); return Similarity(*buffer_reference.get(), buffer_test); } template <typename T> double OpenGLBufferComparison::ComputeTotalSimilarity( const phx::OpenGLImageBufferData<T>& buffer1, const phx::OpenGLImageBufferData<T>& buffer2, double max_distance) { double total_similarity = 0.0; for (std::size_t y = 0; y < buffer1.GetHeight(); ++y) { for (std::size_t x = 0; x < buffer1.GetWidth(); ++x) { total_similarity += ComputePixelSimilarity(buffer1, buffer2, x, y, max_distance); } } return total_similarity; } template <typename T> double OpenGLBufferComparison::ComputePixelSimilarity( const phx::OpenGLImageBufferData<T>& buffer1, const phx::OpenGLImageBufferData<T>& buffer2, std::size_t x, std::size_t y, double max_distance) { const auto pixel1 = buffer1.GetPixel(x, y); const auto pixel2 = buffer2.GetPixel(x, y); const auto dist = PixelDistance(pixel1, pixel2); const double pixel_similarity = 1.0 - (dist / max_distance); return pixel_similarity; } #define INSTANTIATE_COMPUTE_TOTAL_SIMILARITY(T, max_distance_value) \ template <> \ inline double OpenGLBufferComparison::ComputeTotalSimilarity( \ const phx::OpenGLImageBufferData<T>& buffer1, \ const phx::OpenGLImageBufferData<T>& buffer2) { \ return OpenGLBufferComparison::ComputeTotalSimilarity(buffer1, buffer2, \ max_distance_value); \ } INSTANTIATE_COMPUTE_TOTAL_SIMILARITY(phx::OpenGLImageBufferDataType_RGB, sqrt(std::pow(255.0, 2) * 3.0)) INSTANTIATE_COMPUTE_TOTAL_SIMILARITY(phx::OpenGLImageBufferDataType_RGBA, sqrt(std::pow(255.0, 2) * 4.0)) INSTANTIATE_COMPUTE_TOTAL_SIMILARITY(phx::OpenGLImageBufferDataType_Byte, 255.0) INSTANTIATE_COMPUTE_TOTAL_SIMILARITY(phx::OpenGLImageBufferDataType_Float32, 1.0) template <typename T> std::tuple<std::string, std::string, std::string> OpenGLBufferComparison::SaveTemporaryFiles( const phx::OpenGLImageBufferData<T>& buffer_test, phx::Image* ref_image, phx::OpenGLImageBufferData<phx::OpenGLImageBufferDataType_Float32>* buffer_diff) { // create temporary file names std::string randomstring = GenerateRandomString(10); std::string file1 = "buffer_comparison_" + randomstring + "_test.png"; std::string file2 = (ref_image != nullptr ? "buffer_comparison_" + randomstring + "_reference.png" : ""); std::string file_diff = "buffer_comparison_" + randomstring + "_diffmag.png"; // write files there buffer_test.SaveToFilePNG(file1); if (ref_image != nullptr) { ref_image->Save(file2); } if (buffer_diff != nullptr) { buffer_diff->SaveToFilePNG(file_diff); } return std::make_tuple(file1, file2, file_diff); } template <typename T> void OpenGLBufferComparison::OutputFailMessage( const phx::OpenGLImageBufferData<T>& buffer_test, const std::string& filename_reference_image, double similarity, phx::Image* ref_image) { std::cout << std::endl << "=============================================================" "======" << std::endl << "A reference image similarity comparison has FAILED." << std::endl << "=============================================================" "======" << std::endl << "An image buffer " << std::endl << " (" << buffer_test.GetWidth() << " x " << buffer_test.GetHeight() << ", " << T::GetFormatString() << ")" << std::endl << "has been compared against a reference image path" << std::endl << " " << filename_reference_image << std::endl << " (" << (similarity < 0 ? "file does not exist" : std::to_string(ref_image->GetDimensions()[0]) + " x " + std::to_string(ref_image->GetDimensions()[1]) + ", " + GetFormatString(ref_image)) << ")" << std::endl << std::endl; } template <typename T> void OpenGLBufferComparison::DetermineTestOutcome( double similarity, double minimumSimilarity, const phx::OpenGLImageBufferData<T>& buffer_test, const std::string& filename_reference_image, const std::string& temp_file1, const std::string& temp_file2, const std::string& temp_file_diff, bool ref_image_exists) { std::string response = ReadInput(); bool overwrite = ref_image_exists && response == "overwrite"; bool write_new = !ref_image_exists && response == "write"; if (!overwrite && !write_new) { REQUIRE(similarity >= minimumSimilarity); } else { // overwrite reference image with test image buffer_test.CreateImage()->Save(filename_reference_image); OutputUpdateRefImageInfo(filename_reference_image, ref_image_exists); RemoveTempFiles(temp_file1, temp_file2, temp_file_diff); // pass test REQUIRE(true); } } } // namespace test_utilities #endif // TESTS_TEST_UTILITIES_OPENGL_BUFFER_DATA_COMPARISON_HPP_