From 8318c20eb0837aa65b76e199c366a4d4ea777005 Mon Sep 17 00:00:00 2001 From: yuyoyuppe Date: Fri, 9 Apr 2021 16:05:21 +0300 Subject: [PATCH] [VCM] use adaptive jpg quality instead of 0.9 and report more video format info --- .../DirectShowUtils.cpp | 4 + .../ImageLoading.cpp | 56 ++++++----- .../VideoCaptureProxyFilter.cpp | 93 +++++++++++++------ .../VideoConferenceShared/Logging.h | 22 +++++ tools/WebcamReportTool/main.cpp | 14 ++- 5 files changed, 135 insertions(+), 54 deletions(-) diff --git a/src/modules/videoconference/VideoConferenceProxyFilter/DirectShowUtils.cpp b/src/modules/videoconference/VideoConferenceProxyFilter/DirectShowUtils.cpp index feb4cd079e..00359670f3 100644 --- a/src/modules/videoconference/VideoConferenceProxyFilter/DirectShowUtils.cpp +++ b/src/modules/videoconference/VideoConferenceProxyFilter/DirectShowUtils.cpp @@ -22,6 +22,10 @@ unique_media_type_ptr CopyMediaType(const AM_MEDIA_TYPE* source) wil::com_ptr_nothrow GetPinAllocator(wil::com_ptr_nothrow& inputPin) { + if (!inputPin) + { + return nullptr; + } wil::com_ptr_nothrow allocator; if (auto memInput = inputPin.try_query(); memInput) { diff --git a/src/modules/videoconference/VideoConferenceProxyFilter/ImageLoading.cpp b/src/modules/videoconference/VideoConferenceProxyFilter/ImageLoading.cpp index 28568604f8..b665c379cb 100644 --- a/src/modules/videoconference/VideoConferenceProxyFilter/ImageLoading.cpp +++ b/src/modules/videoconference/VideoConferenceProxyFilter/ImageLoading.cpp @@ -21,31 +21,12 @@ #include #include #include +#include #include #include "Logging.h" -bool failed(HRESULT hr) -{ - return hr != S_OK; -} - -bool failed(bool val) -{ - return val == false; -} - -template -bool failed(wil::com_ptr_nothrow& ptr) -{ - return ptr == nullptr; -} - -#define OK_OR_BAIL(expr) \ - if (failed(expr)) \ - return {}; - IWICImagingFactory* _GetWIC() noexcept { static IWICImagingFactory* s_Factory = nullptr; @@ -110,7 +91,8 @@ wil::com_ptr_nothrow EncodeBitmapToContainer(IWICImagingFactory* pWIC, wil::com_ptr_nothrow bitmap, const GUID& containerGUID, const UINT width, - const UINT height) + const UINT height, + const float quality) { wil::com_ptr_nothrow encoder; pWIC->CreateEncoder(containerGUID, nullptr, &encoder); @@ -125,8 +107,31 @@ wil::com_ptr_nothrow EncodeBitmapToContainer(IWICImagingFactory* pWIC, OK_OR_BAIL(CreateStreamOnHGlobal(nullptr, true, &encodedBitmap)); encoder->Initialize(encodedBitmap.get(), WICBitmapEncoderNoCache); wil::com_ptr_nothrow encodedFrame; - OK_OR_BAIL(encoder->CreateNewFrame(&encodedFrame, nullptr)); - OK_OR_BAIL(encodedFrame->Initialize(nullptr)); + + wil::com_ptr_nothrow encoderOptions; + OK_OR_BAIL(encoder->CreateNewFrame(&encodedFrame, &encoderOptions)); + + ULONG nProperties = 0; + OK_OR_BAIL(encoderOptions->CountProperties(&nProperties)); + for (ULONG propIdx = 0; propIdx < nProperties; ++propIdx) + { + PROPBAG2 propBag{}; + ULONG _; + OK_OR_BAIL(encoderOptions->GetPropertyInfo(propIdx, 1, &propBag, &_)); + if (propBag.pstrName == std::wstring_view{ L"ImageQuality" }) + { + wil::unique_variant variant; + variant.vt = VT_R4; + variant.fltVal = quality; + OK_OR_BAIL(encoderOptions->Write(1, &propBag, &variant)); + LOG("Successfully set jpg compression quality"); + // skip the rest of the properties + propIdx = nProperties; + } + CoTaskMemFree(propBag.pstrName); + } + + OK_OR_BAIL(encodedFrame->Initialize(encoderOptions.get())); WICPixelFormatGUID intermediateFormat = GUID_WICPixelFormat24bppRGB; OK_OR_BAIL(encodedFrame->SetPixelFormat(&intermediateFormat)); @@ -245,7 +250,8 @@ IMFSample* ConvertIMFVideoSample(const MFT_REGISTER_TYPE_INFO& inputType, } wil::com_ptr_nothrow LoadImageAsSample(wil::com_ptr_nothrow imageStream, - IMFMediaType* sampleMediaType) noexcept + IMFMediaType* sampleMediaType, + const float quality) noexcept { UINT targetWidth = 0; UINT targetHeight = 0; @@ -302,7 +308,7 @@ wil::com_ptr_nothrow LoadImageAsSample(wil::com_ptr_nothrow { // Use an intermediate jpg container sample which will be transcoded to the target format wil::com_ptr_nothrow jpgStream = - EncodeBitmapToContainer(pWIC, srcImageBitmap, GUID_ContainerFormatJpeg, targetWidth, targetHeight); + EncodeBitmapToContainer(pWIC, srcImageBitmap, GUID_ContainerFormatJpeg, targetWidth, targetHeight, quality); // Obtain stream size and lock its memory pointer STATSTG intermediateStreamStat{}; diff --git a/src/modules/videoconference/VideoConferenceProxyFilter/VideoCaptureProxyFilter.cpp b/src/modules/videoconference/VideoConferenceProxyFilter/VideoCaptureProxyFilter.cpp index 05077de865..bdda7514cc 100644 --- a/src/modules/videoconference/VideoConferenceProxyFilter/VideoCaptureProxyFilter.cpp +++ b/src/modules/videoconference/VideoConferenceProxyFilter/VideoCaptureProxyFilter.cpp @@ -35,7 +35,8 @@ wil::com_ptr_nothrow VideoCaptureProxyPin::FindAllocator() } wil::com_ptr_nothrow LoadImageAsSample(wil::com_ptr_nothrow imageStream, - IMFMediaType* sampleMediaType) noexcept; + IMFMediaType* sampleMediaType, + const float quality) noexcept; HRESULT VideoCaptureProxyPin::Connect(IPin* pReceivePin, const AM_MEDIA_TYPE*) { @@ -69,6 +70,7 @@ HRESULT VideoCaptureProxyPin::Connect(IPin* pReceivePin, const AM_MEDIA_TYPE*) } auto allocator = FindAllocator(); + memInput->NotifyAllocator(allocator.get(), false); return S_OK; @@ -349,6 +351,20 @@ HRESULT VideoCaptureProxyPin::QuerySupported(REFGUID guidPropSet, DWORD dwPropID return S_OK; } +long GetImageSize(wil::com_ptr_nothrow& image) +{ + if (!image) + { + return 0; + } + DWORD imageSize = 0; + wil::com_ptr_nothrow imageBuf; + + OK_OR_BAIL(image->GetBufferByIndex(0, &imageBuf)); + OK_OR_BAIL(imageBuf->GetCurrentLength(&imageSize)); + return imageSize; +} + void OverwriteFrame(IMediaSample* frame, wil::com_ptr_nothrow& image) { if (!image) @@ -365,17 +381,23 @@ void OverwriteFrame(IMediaSample* frame, wil::com_ptr_nothrow& image) } wil::com_ptr_nothrow imageBuf; - const long nBytes = frame->GetSize(); + const DWORD frameSize = frame->GetSize(); image->GetBufferByIndex(0, &imageBuf); BYTE* imageData = nullptr; - DWORD maxLength = 0, curLength = 0; - imageBuf->Lock(&imageData, &maxLength, &curLength); - std::copy(imageData, imageData + curLength, frameData); - imageBuf->Unlock(); - frame->SetActualDataLength(curLength); -} + DWORD _ = 0, imageSize = 0; + imageBuf->Lock(&imageData, &_, &imageSize); + if (imageSize > frameSize) + { + LOG("Error: overlay image size is larger than a frame size - truncated."); + imageSize = frameSize; + } + + std::copy(imageData, imageData + imageSize, frameData); + imageBuf->Unlock(); + frame->SetActualDataLength(imageSize); +} VideoCaptureProxyFilter::VideoCaptureProxyFilter() : _worker_thread{ std::thread{ [this]() { @@ -566,12 +588,18 @@ HRESULT VideoCaptureProxyFilter::EnumPins(IEnumPins** ppEnum) } auto& webcam = webcams[*selectedCamIdx]; - auto pin = winrt::make_self(); pin->_mediaFormat = CopyMediaType(webcam.bestFormat.mediaType); pin->_owningFilter = this; _outPin.attach(pin.detach()); + auto frameCallback = [this](IMediaSample* sample) { + std::unique_lock lock{ _worker_mutex }; + sample->AddRef(); + _pending_frame = sample; + _worker_cv.notify_one(); + }; + wil::com_ptr_nothrow targetMediaType; MFCreateMediaType(&targetMediaType); targetMediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); @@ -582,28 +610,37 @@ HRESULT VideoCaptureProxyFilter::EnumPins(IEnumPins** ppEnum) targetMediaType.get(), MF_MT_FRAME_SIZE, webcam.bestFormat.width, webcam.bestFormat.height); MFSetAttributeRatio(targetMediaType.get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1); - if (!_blankImage) - { - wil::com_ptr_nothrow blackBMPImage = SHCreateMemStream(bmpPixelData, sizeof(bmpPixelData)); - _blankImage = LoadImageAsSample(blackBMPImage, targetMediaType.get()); - } - - if (newSettings.overlayImage && !_overlayImage) - { - _overlayImage = LoadImageAsSample(newSettings.overlayImage, targetMediaType.get()); - } - - LOG("Loaded images"); - auto frameCallback = [this](IMediaSample* sample) { - std::unique_lock lock{ _worker_mutex }; - sample->AddRef(); - _pending_frame = sample; - _worker_cv.notify_one(); - }; - _captureDevice = VideoCaptureDevice::Create(std::move(webcam), std::move(frameCallback)); if (_captureDevice) { + if (!_blankImage) + { + wil::com_ptr_nothrow blackBMPImage = SHCreateMemStream(bmpPixelData, sizeof(bmpPixelData)); + _blankImage = LoadImageAsSample(blackBMPImage, targetMediaType.get(), false); + } + + if (newSettings.overlayImage && !_overlayImage) + { + long maxFrameSize = 0; + ALLOCATOR_PROPERTIES allocatorProperties{}; + if (!failed(_captureDevice->_allocator->GetProperties(&allocatorProperties))) + { + maxFrameSize = allocatorProperties.cbBuffer; + } + + size_t selectedModeIdx = 0; + constexpr std::array jpgQualityModes = { 0.5f, 0.25f, 0.1f }; + _overlayImage = LoadImageAsSample(newSettings.overlayImage, targetMediaType.get(), jpgQualityModes[selectedModeIdx]); + long imageSize = GetImageSize(_overlayImage); + while (maxFrameSize && maxFrameSize < imageSize && selectedModeIdx < size(jpgQualityModes)) + { + _overlayImage = LoadImageAsSample(newSettings.overlayImage, targetMediaType.get(), jpgQualityModes[++selectedModeIdx]); + imageSize = GetImageSize(_overlayImage); + } + } + + LOG("Loaded images"); + LOG("Capture device created successfully"); } else diff --git a/src/modules/videoconference/VideoConferenceShared/Logging.h b/src/modules/videoconference/VideoConferenceShared/Logging.h index 2893e3d086..633656f695 100644 --- a/src/modules/videoconference/VideoConferenceShared/Logging.h +++ b/src/modules/videoconference/VideoConferenceShared/Logging.h @@ -4,6 +4,9 @@ #include #include +#include +#include + void LogToFile(std::string what, const bool verbose = false); void LogToFile(std::wstring what, const bool verbose = false); std::string toMediaTypeString(GUID subtype); @@ -38,3 +41,22 @@ std::string toMediaTypeString(GUID subtype); #define LOG(str) LogToFile(str, false); #endif +inline bool failed(HRESULT hr) +{ + return hr != S_OK; +} + +inline bool failed(bool val) +{ + return val == false; +} + +template +inline bool failed(wil::com_ptr_nothrow& ptr) +{ + return ptr == nullptr; +} + +#define OK_OR_BAIL(expr) \ + if (failed(expr)) \ + return {}; diff --git a/tools/WebcamReportTool/main.cpp b/tools/WebcamReportTool/main.cpp index 06e25ab2f8..89ff3e8cda 100644 --- a/tools/WebcamReportTool/main.cpp +++ b/tools/WebcamReportTool/main.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -90,6 +91,15 @@ std::string GetMediaSubTypeString(const GUID& guid) } } +std::string asString(RECT r) +{ + std::ostringstream s; + if (r.left == r.right && r.bottom == r.right && r.top == r.right && r.right == 0) + return "[ZEROES]"; + s << '(' << r.left << ", " << r.top << ") - (" << r.right << ", " << r.bottom << ")"; + return s.str(); +} + void LogMediaTypes(wil::com_ptr_nothrow& pin) { wil::com_ptr_nothrow mediaTypeEnum; @@ -113,7 +123,9 @@ void LogMediaTypes(wil::com_ptr_nothrow& pin) } const auto formatAvgFPS = 10000000LL / format->AvgTimePerFrame; log() << GetMediaSubTypeString(mt->subtype) << '\t' << format->bmiHeader.biWidth << "x" - << format->bmiHeader.biHeight << " - " << formatAvgFPS << "fps\n"; + << format->bmiHeader.biHeight << " - " << formatAvgFPS << "fps src rect: " << asString(format->rcSource) + << " dst rect: " << asString(format->rcSource) + << " flipped: " << std::boolalpha << (format->bmiHeader.biHeight < 0) << '\n'; } log() << '\n'; }