From 1769165661a3849f1cca7934d377c53c58575207 Mon Sep 17 00:00:00 2001 From: "zejian.su" Date: Tue, 5 Dec 2023 10:43:10 +0800 Subject: [PATCH] Fix the sync issue between the Preview.Show() and the QWidget.update() 1. Add a mutex between the Preview.Show() and the QWidget.update(). 2. Speed up the Preview.Show() -- change the floating point to integer, merge some plus and multiplying operation and remove some data copy. Signed-off-by: zejian.su --- ...ue-between-the-Preview.Show-and-the-.patch | 291 ++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 package/starfive/libcamera-apps/0009-Fix-the-sync-issue-between-the-Preview.Show-and-the-.patch diff --git a/package/starfive/libcamera-apps/0009-Fix-the-sync-issue-between-the-Preview.Show-and-the-.patch b/package/starfive/libcamera-apps/0009-Fix-the-sync-issue-between-the-Preview.Show-and-the-.patch new file mode 100644 index 00000000..bf10b5e6 --- /dev/null +++ b/package/starfive/libcamera-apps/0009-Fix-the-sync-issue-between-the-Preview.Show-and-the-.patch @@ -0,0 +1,291 @@ +From e9e9555cac3c11c70277824d82d5b3fbd5e4afd2 Mon Sep 17 00:00:00 2001 +From: "zejian.su" +Date: Fri, 1 Dec 2023 16:30:27 +0800 +Subject: [PATCH 2/2] Fix the sync issue between the Preview.Show() and the + QWidget.update() 1. Add a mutex between the Preview.Show() and the + QWidget.update(). 2. Speed up the Preview.Show() -- change the floating point + to integer, merge some plus and multiplying operation and remove some data + copy. + +Signed-off-by: zejian.su +--- + preview/qt_preview.cpp | 183 +++++++++++++++++++++++++---------------- + 1 file changed, 113 insertions(+), 70 deletions(-) + +diff --git a/preview/qt_preview.cpp b/preview/qt_preview.cpp +index 8196a8e..ef3c8a4 100644 +--- a/preview/qt_preview.cpp ++++ b/preview/qt_preview.cpp +@@ -41,16 +41,47 @@ class MyWidget : public QWidget + public: + MyWidget(QWidget *parent, int w, int h) : QWidget(parent), size(w, h) + { +- image = QImage(size, QImage::Format_RGB888); +- image.fill(0); ++ for(int i = 0; i < 2; i++) { ++ buffers_[i].image = QImage(size, QImage::Format_RGB888); ++ buffers_[i].image.fill(0); ++ availableBuffers_.push_back(&buffers_[i]); ++ } + } + QSize size; +- QImage image; ++ ++ struct ImageBuffer { ++ QImage image; ++ uint8_t frameCounter; ++ ++ ImageBuffer() : frameCounter(0) {}; ++ } buffers_[2]; ++ ++ std::list freeBuffers_; ++ std::list availableBuffers_; ++ ++ std::mutex buffers_available_mutex_; ++ std::mutex buffers_free_mutex_; + protected: + void paintEvent(QPaintEvent *) override + { + QPainter painter(this); +- painter.drawImage(rect(), image, image.rect()); ++ { ++ ImageBuffer *buffer = nullptr; ++ { ++ std::lock_guard lock(buffers_available_mutex_); ++ if(!availableBuffers_.size()) ++ return; ++ buffer = availableBuffers_.back(); ++ availableBuffers_.pop_back(); ++ } ++ ++ painter.drawImage(rect(), buffer->image, buffer->image.rect()); ++ ++ { ++ std::lock_guard lock(buffers_free_mutex_); ++ freeBuffers_.push_back(buffer); ++ } ++ } + } + QSize sizeHint() const override { return size; } + }; +@@ -58,7 +89,7 @@ protected: + class QtPreview : public Preview + { + public: +- QtPreview(Options const *options) : Preview(options), frame_counter_(0) ++ QtPreview(Options const *options) : Preview(options) + { + window_width_ = options->preview_width; + window_height_ = options->preview_height; +@@ -67,6 +98,8 @@ public: + // This preview window is expensive, so make it small by default. + if (window_width_ == 0 || window_height_ == 0) + window_width_ = 512, window_height_ = 384; ++ ++ frameCounter_ = 0; + // As a hint, reserve twice the binned width for our widest current camera (V3) + tmp_stripe_.reserve(4608); + thread_ = std::thread(&QtPreview::threadFunc, this, options); +@@ -83,100 +116,105 @@ public: + void SetInfoText(const std::string &text) override { main_window_->setWindowTitle(QString::fromStdString(text)); } + virtual void Show(int fd, libcamera::Span span, StreamInfo const &info) override + { +- if((frame_counter_++) & 1) { +- // Return the buffer to the camera system. +- done_callback_(fd); +- return; ++ MyWidget::ImageBuffer *buffer = nullptr; ++ { ++ std::lock_guard lock(pane_->buffers_free_mutex_); ++ if(pane_->freeBuffers_.size() > 0) { ++ buffer = pane_->freeBuffers_.front(); ++ pane_->freeBuffers_.pop_front(); ++ } ++ } ++ ++ if(!buffer) { ++ std::lock_guard lock(pane_->buffers_available_mutex_); ++ if(!pane_->availableBuffers_.size()) { ++ done_callback_(fd); ++ return; ++ } ++ buffer = pane_->availableBuffers_.front(); ++ pane_->availableBuffers_.pop_front(); + } +- + // Quick and simple nearest-neighbour-ish resampling is used here. + // We further share U,V samples between adjacent output pixel pairs + // (even when downscaling) to speed up the conversion. + unsigned x_step = (info.width << 16) / window_width_; + unsigned y_step = (info.height << 16) / window_height_; + +- // Choose the right matrix to convert YUV back to RGB. +- static const float YUV2RGB[3][9] = { +- { 1.0, 0.0, 1.402, 1.0, -0.344, -0.714, 1.0, 1.772, 0.0 }, // JPEG +- { 1.164, 0.0, 1.596, 1.164, -0.392, -0.813, 1.164, 2.017, 0.0 }, // SMPTE170M +- { 1.164, 0.0, 1.793, 1.164, -0.213, -0.533, 1.164, 2.112, 0.0 }, // Rec709 ++ static const uint32_t YUV2RGB[3][9] = { ++ { 128, 0, 179, 128, 44, 91, 128, 227, 0 }, // JPEG ++ { 149, 0, 204, 149, 50, 104, 149, 258, 0 }, // SMPTE170M ++ { 149, 0, 230, 149, 27, 68, 149, 270, 0 }, // Rec709 + }; +- int offsetY; +- float coeffY, coeffVR, coeffUG, coeffVG, coeffUB; +- if (info.colour_space == libcamera::ColorSpace::Smpte170m) ++ static const int RGBOFFSET[3][3] = { ++ {24960, -15232, 31104}, {28496, -17328, 35408}, {31824, -9776, 36944} ++ }; ++ ++ uint32_t coeffY, coeffVR, coeffUG, coeffVG, coeffUB; ++ const int * rgbOffset = nullptr; ++ if(info.colour_space == libcamera::ColorSpace::Smpte170m) + { +- offsetY = 16; + coeffY = YUV2RGB[1][0]; + coeffVR = YUV2RGB[1][2]; + coeffUG = YUV2RGB[1][4]; + coeffVG = YUV2RGB[1][5]; + coeffUB = YUV2RGB[1][7]; +- } +- else if (info.colour_space == libcamera::ColorSpace::Rec709) ++ rgbOffset = RGBOFFSET[1]; ++ } else if(info.colour_space == libcamera::ColorSpace::Rec709) + { +- offsetY = 16; + coeffY = YUV2RGB[2][0]; + coeffVR = YUV2RGB[2][2]; + coeffUG = YUV2RGB[2][4]; + coeffVG = YUV2RGB[2][5]; + coeffUB = YUV2RGB[2][7]; +- } +- else ++ rgbOffset = RGBOFFSET[2]; ++ } else + { +- offsetY = 0; + coeffY = YUV2RGB[0][0]; + coeffVR = YUV2RGB[0][2]; + coeffUG = YUV2RGB[0][4]; + coeffVG = YUV2RGB[0][5]; + coeffUB = YUV2RGB[0][7]; +- if (info.colour_space != libcamera::ColorSpace::Sycc) +- LOG(1, "QtPreview: unexpected colour space " << libcamera::ColorSpace::toString(info.colour_space)); ++ rgbOffset = RGBOFFSET[0]; + } + +- // Because the source buffer is uncached, and we want to read it a byte at a time, +- // take a copy of each row used. This is a speedup provided memcpy() is vectorized. +- tmp_stripe_.resize(2 * info.stride); +- uint8_t const *Y_start = span.data(); +- uint8_t const *UV_start = Y_start + info.height * info.stride; +- uint8_t *Y_row = &tmp_stripe_[0]; +- uint8_t *U_row = Y_row + info.stride; +- uint8_t *V_row = U_row + (info.stride >> 1); +- +- // Possibly this should be locked in case a repaint is happening? In practice the risk +- // is only that there might be some tearing, so I don't think we worry. We could speed +- // it up by getting the ISP to supply RGB, but I'm not sure I want to handle that extra +- // possibility in our main application code, so we'll put up with the slow conversion. +- for (unsigned int y = 0; y < window_height_; y++) ++ uint8_t const * Y_start = span.data(); ++ uint8_t const * UV_start = Y_start + info.height * info.stride; ++ int src_ypos = y_step >> 1; ++ uint32_t Y2 = 0; ++ uint32_t U2 = coeffUG | (coeffUB << 16); ++ uint32_t V2 = coeffVR | (coeffVG << 16); ++ uint16_t * y2 = (uint16_t *)&Y2; ++ uint32_t U; ++ uint32_t V; ++ uint16_t * u2 = (uint16_t *)&U; ++ uint16_t * v2 = (uint16_t *)&V; ++ ++ for(unsigned int y = 0; y < window_height_; y++, src_ypos += y_step) + { +- unsigned row = (y * y_step) >> 16; +- uint8_t *dest = pane_->image.scanLine(y); +- unsigned x_pos = x_step >> 1; +- +- memcpy(Y_row, Y_start + row * info.stride, info.stride); +- //memcpy(U_row, Y_start + ((4 * info.height + row) >> 1) * (info.stride >> 1), info.stride >> 1); +- //memcpy(V_row, Y_start + ((5 * info.height + row) >> 1) * (info.stride >> 1), info.stride >> 1); +- uint8_t const *cur_uv = UV_start + (row >> 1) * info.width; +- for(unsigned int uv_idx = 0; uv_idx < info.width >> 1; uv_idx++, cur_uv += 2) +- U_row[uv_idx] = cur_uv[0], V_row[uv_idx] = cur_uv[1]; +- +- for (unsigned int x = 0; x < window_width_; x += 2) ++ const uint8_t * src_y = Y_start + (src_ypos >> 16) * info.width; ++ const uint8_t * src_uv = UV_start + (src_ypos >> 17) * info.width; ++ uint8_t * dest = buffer->image.scanLine(y); ++ uint32_t x_pos = x_step >> 1; ++ ++ for(unsigned int x = 0; x < window_width_; x += 2) + { +- int Y0 = Y_row[x_pos >> 16]; ++ y2[0] = src_y[x_pos >> 16]; + x_pos += x_step; +- int Y1 = Y_row[x_pos >> 16]; +- int U = U_row[x_pos >> 17]; +- int V = V_row[x_pos >> 17]; ++ y2[1] = src_y[x_pos >> 16]; ++ U = src_uv[(x_pos >> 16) & 0xfffffffe]; ++ V = src_uv[(x_pos >> 16) | 1]; + x_pos += x_step; +- Y0 -= offsetY; +- Y1 -= offsetY; +- U -= 128; +- V -= 128; +- int R0 = coeffY * Y0 + coeffVR * V; +- int G0 = coeffY * Y0 + coeffUG * U + coeffVG * V; +- int B0 = coeffY * Y0 + coeffUB * U; +- int R1 = coeffY * Y1 + coeffVR * V; +- int G1 = coeffY * Y1 + coeffUG * U + coeffVG * V; +- int B1 = coeffY * Y1 + coeffUB * U; ++ ++ Y2 *= coeffY; ++ U *= U2; ++ V *= V2; ++ ++ int R0 = ((int)y2[0] + (int)v2[0] - rgbOffset[0]) >> 7; ++ int G0 = ((int)y2[0] - (int)u2[0] - (int)v2[1] - rgbOffset[1]) >> 7; ++ int B0 = ((int)y2[0] + (int)u2[1] - rgbOffset[2]) >> 7; ++ int R1 = ((int)y2[1] + (int)v2[0] - rgbOffset[0]) >> 7; ++ int G1 = ((int)y2[1] - (int)u2[0] - (int)v2[1] - rgbOffset[1]) >> 7; ++ int B1 = ((int)y2[1] + (int)u2[1] - rgbOffset[2]) >> 7; + *(dest++) = std::clamp(R0, 0, 255); + *(dest++) = std::clamp(G0, 0, 255); + *(dest++) = std::clamp(B0, 0, 255); +@@ -186,6 +224,12 @@ public: + } + } + ++ { ++ std::lock_guard lock(pane_->buffers_available_mutex_); ++ buffer->frameCounter = ++frameCounter_; ++ pane_->availableBuffers_.push_back(buffer); ++ } ++ + pane_->update(); + + // Return the buffer to the camera system. +@@ -193,7 +237,7 @@ public: + } + // Reset the preview window, clearing the current buffers and being ready to + // show new ones. +- void Reset() override {frame_counter_ = 0;} ++ void Reset() override {} + // Check if preview window has been shut down. + bool Quit() override { return main_window_->quit; } + // There is no particular limit to image sizes, though large images will be very slow. +@@ -228,8 +272,7 @@ private: + std::mutex mutex_; + std::condition_variable cond_var_; + std::vector tmp_stripe_; +- +- unsigned int frame_counter_; ++ uint8_t frameCounter_; + }; + + Preview *make_qt_preview(Options const *options) +-- +2.34.1 +