diff --git a/HPPA/DepthCameraWindow.cpp b/HPPA/DepthCameraWindow.cpp index a1f9ef2..67109e3 100644 --- a/HPPA/DepthCameraWindow.cpp +++ b/HPPA/DepthCameraWindow.cpp @@ -305,10 +305,14 @@ void DepthCameraOperation::OpenDepthCamera_getDepthValue() auto vid = devInfo->getVid(); config->enableVideoStream(OB_STREAM_DEPTH, 640, 480, 15, OB_FORMAT_Y16); + config->enableVideoStream(OB_STREAM_COLOR, 640, 480, 15, OB_FORMAT_YUYV); config->setFrameAggregateOutputMode(OB_FRAME_AGGREGATE_OUTPUT_ALL_TYPE_FRAME_REQUIRE); m_pipe->enableFrameSync(); + // Create a format converter filter. + auto formatConverter = std::make_shared(); + m_pipe->start(config); // Drop several frames @@ -322,7 +326,10 @@ void DepthCameraOperation::OpenDepthCamera_getDepthValue() record = true; QString fileNamePrefix = AppSettings::instance().depthCameraDataFolder() + QDir::separator() + "Gemini336L"; - double depthValue_all = 0.0; + // 增量平均所需的变量 + cv::Mat avgRgbMat, avgDepthMat; + int avgFrameCount = 0; + for (size_t i = 0; i < m_averageNumberOfTimes; i++) { if(frameIndex==0) @@ -342,25 +349,56 @@ void DepthCameraOperation::OpenDepthCamera_getDepthValue() // 彩色和深度图像 auto depthFrame = frameSet->getFrame(OB_FRAME_DEPTH)->as(); + auto colorFrame = frameSet->getFrame(OB_FRAME_COLOR)->as(); - //是否需要保存深度图像？？？？？？？？ + // Convert the color frame to RGB format. + if (colorFrame->format() != OB_FORMAT_RGB) { + if (colorFrame->format() == OB_FORMAT_MJPG) { + formatConverter->setFormatConvertType(FORMAT_MJPG_TO_RGB); + } + else if (colorFrame->format() == OB_FORMAT_UYVY) { + formatConverter->setFormatConvertType(FORMAT_UYVY_TO_RGB); + } + else if (colorFrame->format() == OB_FORMAT_YUYV) { + formatConverter->setFormatConvertType(FORMAT_YUYV_TO_RGB); + } + else { + std::cout << "Color format is not support!" << std::endl; + continue; + } + colorFrame = formatConverter->process(colorFrame)->as(); + } + // Processed the color frames to BGR format, use OpenCV to save to disk. + formatConverter->setFormatConvertType(FORMAT_RGB_TO_BGR); + colorFrame = formatConverter->process(colorFrame)->as(); + + //用于测试：保存深度图像 //saveDepthFrame(depthFrame, frameIndex, fileNamePrefix.toStdString()); + //saveColorFrame(colorFrame, frameIndex, fileNamePrefix.toStdString()); + + cv::Mat colorMat(colorFrame->height(), colorFrame->width(), CV_8UC3, colorFrame->data()); + cv::Mat rgbMat; + cv::cvtColor(colorMat, rgbMat, cv::COLOR_BGR2RGB); + //m_colorImage = QImage(rgbMat.data, rgbMat.cols, rgbMat.rows, static_cast(rgbMat.step), QImage::Format_RGB888).copy(); + cv::Mat depthMat(depthFrame->height(), depthFrame->width(), CV_16UC1, depthFrame->data()); - //裁剪边缘区域 - int cropRows = static_cast(depthMat.rows * (1 - m_percentageOfEffectiveArea) / 2); - int cropCols = static_cast(depthMat.cols * (1 - m_percentageOfEffectiveArea) / 2); - cv::Rect roi(cropCols, cropRows, - depthMat.cols - 2 * cropCols, - depthMat.rows - 2 * cropRows); - cv::Mat depthRoi = depthMat(roi); + // 增量平均计算 + if (avgFrameCount == 0) + { + avgRgbMat = cv::Mat::zeros(rgbMat.size(), CV_32FC3); + avgDepthMat = cv::Mat::zeros(depthMat.size(), CV_32F); + } + avgFrameCount++; + cv::Mat rgbFloat; + rgbMat.convertTo(rgbFloat, CV_32FC3); + avgRgbMat = avgRgbMat + (rgbFloat - avgRgbMat) / avgFrameCount; + + cv::Mat depthMatTmp; + depthMat.convertTo(depthMatTmp, CV_32F); + avgDepthMat = avgDepthMat + (depthMatTmp - avgDepthMat) / avgFrameCount; - //计算平均深度值并累加 - cv::Scalar meanDepth = cv::mean(depthRoi); - double depthValue = meanDepth[0] / 1000.0; // 转换为米 - depthValue_all += depthValue; - std::cout << "Depth value: " << depthValue << " m, accumulated: " << depthValue_all << std::endl; cv::Mat depthMat8U; depthMat.convertTo(depthMat8U, CV_8UC1, 255.0 / 4096.0); @@ -375,13 +413,51 @@ void DepthCameraOperation::OpenDepthCamera_getDepthValue() frameIndex++; } + m_pipe->stop(); + + // 对累积平均后的图像进行处理 + double depthValue; + if (avgFrameCount > 0) + { + cv::Mat avgRgbResult, avgDepthResult; + avgRgbMat.convertTo(avgRgbResult, CV_8UC3); + avgDepthMat.convertTo(avgDepthResult, CV_16UC1); + + // 保存平均结果图像 + std::vector pngParams; + pngParams.push_back(cv::IMWRITE_PNG_COMPRESSION); + pngParams.push_back(0); + pngParams.push_back(cv::IMWRITE_PNG_STRATEGY); + pngParams.push_back(cv::IMWRITE_PNG_STRATEGY_DEFAULT); + cv::imwrite(getTestFilePath("_AvgRGB_").toStdString(), avgRgbResult, pngParams); + cv::imwrite(getTestFilePath("_AvgDepth_").toStdString(), avgDepthResult, pngParams); + + // 创建掩膜，排除深度值为0的区域 + cv::Mat mask = avgDepthResult != 0; + + // 保存掩膜 + cv::Mat mask8U; + mask.convertTo(mask8U, CV_8UC1, 255.0); + std::string maskName = fileNamePrefix.toStdString() + "_Mask_" + std::to_string(mask.cols) + "x" + std::to_string(mask.rows) + ".png"; + cv::imwrite(maskName, mask8U, pngParams); + + if (m_depthAlgorithm == 0) + { + depthValue = processAveragedImages_roiAvg(avgDepthResult, mask); + } + else if (m_depthAlgorithm == 1) + { + depthValue = processAveragedImages_depthRangePercentage(avgDepthResult, mask); + } + else if (m_depthAlgorithm == 2) + { + depthValue = processAveragedImages_segmentation(avgRgbResult, avgDepthResult, mask); + } + } //计算平均深度值 - double depthValue_avg = depthValue_all / m_averageNumberOfTimes; - std::cout << "Average depth value: " << depthValue_avg << " m" << std::endl; - emit DepthValueSignal(depthValue_avg); - - m_pipe->stop(); + std::cout << "Depth value: " << depthValue << " m" << std::endl; + emit DepthValueSignal(depthValue); delete m_pipe; m_pipe = nullptr; @@ -389,6 +465,97 @@ void DepthCameraOperation::OpenDepthCamera_getDepthValue() record = false; } +double DepthCameraOperation::processAveragedImages_roiAvg(const cv::Mat& avgDepthResult, const cv::Mat& mask) +{ + //裁剪边缘区域 + int cropRows = static_cast(avgDepthResult.rows * (1 - m_percentageOfEffectiveArea) / 2); + int cropCols = static_cast(avgDepthResult.cols * (1 - m_percentageOfEffectiveArea) / 2); + cv::Rect roi(cropCols, cropRows, + avgDepthResult.cols - 2 * cropCols, + avgDepthResult.rows - 2 * cropRows); + cv::Mat depthRoi = avgDepthResult(roi); + cv::Mat maskRoi = mask(roi); + //计算平均深度值，使用掩膜排除深度值为0的区域 + cv::Scalar meanDepth = cv::mean(depthRoi, maskRoi); + double depthValue = meanDepth[0] / 1000.0; // 转换为米 + + return depthValue; +} + +double DepthCameraOperation::processAveragedImages_depthRangePercentage(const cv::Mat& avgDepthResult, const cv::Mat& mask) +{ + // 找出最大最小值 + double minVal, maxVal; + cv::minMaxLoc(avgDepthResult, &minVal, &maxVal, nullptr, nullptr, mask); + + // 检查是否有有效数据 + if (minVal == std::numeric_limits::max()) + { + std::cout << "Warning: No valid depth pixels found in masked region!" << std::endl; + return 0.0; + } + + // 返回最小值乘以m_depthRangePercentage + double depthValue = minVal * m_depthRangePercentage / 1000.0; // 转换为米 + + return depthValue; +} + +double DepthCameraOperation::processAveragedImages_segmentation(const cv::Mat& avgRgbResult, const cv::Mat& avgDepthResult, const cv::Mat& mask) +{ + // 转换到 HSV 颜色空间进行植被分割 + cv::Mat hsvMat; + cv::cvtColor(avgRgbResult, hsvMat, cv::COLOR_RGB2HSV); + + // 分离通道 + std::vector hsvChannels; + cv::split(hsvMat, hsvChannels); + cv::Mat hue = hsvChannels[0]; + cv::Mat sat = hsvChannels[1]; + cv::Mat val = hsvChannels[2]; + + // 定义绿色植被的Hue范围 (OpenCV中Hue范围是0-180，实际绿色约35-90度) + // 扩展范围以覆盖不同光照条件下的绿色 + cv::Mat hueMask1 = (hue >= 35) & (hue <= 85); + cv::Mat satMask = sat > 30; // 饱和度阈值，去除灰色区域 + cv::Mat valMask = val > 50; // 亮度阈值，去除过暗区域 + + // 组合条件生成植被掩膜 + cv::Mat vmask; + cv::bitwise_and(hueMask1, satMask, vmask); + cv::bitwise_and(vmask, valMask, vmask); + + // 结合深度掩膜，计算两个mask的交集 + cv::Mat combinedMask; + cv::bitwise_and(mask, vmask, combinedMask); + + // 保存 vmask 和 combinedMask 到 exe 所在文件夹的文件夹 + cv::Mat vmask8U, combinedMask8U; + vmask.convertTo(vmask8U, CV_8UC1, 255.0); + combinedMask.convertTo(combinedMask8U, CV_8UC1, 255.0); + cv::imwrite(getTestFilePath("vmask").toStdString(), vmask8U); + cv::imwrite(getTestFilePath("combinedMask").toStdString(), combinedMask8U); + + // 计算 avgRgbResult 在 combinedMask 区域内的平均深度值 + double depthValue = 0.0; + if (cv::countNonZero(combinedMask) > 0) { + cv::Scalar meanDepth = cv::mean(avgDepthResult, combinedMask); + depthValue = meanDepth[0] / 1000.0; // 转换为米 + } else { + std::cout << "Warning: No valid pixels in combined mask!" << std::endl; + } + + return depthValue; +} + +QString DepthCameraOperation::getTestFilePath(const QString& fileName) +{ + QString testFolder = QCoreApplication::applicationDirPath() + QDir::separator() + "depthValueTest"; + QDir().mkpath(testFolder); + QString timestamp = QString::number(QDateTime::currentMSecsSinceEpoch()); + return testFolder + QDir::separator() + fileName + "_" + timestamp + ".png"; +} + void DepthCameraOperation::saveDepthFrame(const std::shared_ptr depthFrame, const uint32_t frameIndex, std::string fileNamePrefix_) { std::vector params; diff --git a/HPPA/DepthCameraWindow.h b/HPPA/DepthCameraWindow.h index 93bb823..ae6741b 100644 --- a/HPPA/DepthCameraWindow.h +++ b/HPPA/DepthCameraWindow.h @@ -5,8 +5,10 @@ #include #include #include -#include +#include #include +#include +#include //#include #include @@ -35,8 +37,10 @@ public: void setCaptureInterval(int captureIntervalSeconds); + void setDepthAlgorithm(int depthAlgorithm) { m_depthAlgorithm = depthAlgorithm; } void setAverageNumberOfTimes(double averageNumberOfTimes) { m_averageNumberOfTimes = averageNumberOfTimes; } void setPercentageOfEffectiveArea(double percentageOfEffectiveArea) { m_percentageOfEffectiveArea = percentageOfEffectiveArea; } + void setDepthRangePercentage(double depthRangePercentage) { m_depthRangePercentage = depthRangePercentage; } private: ob::Pipeline* m_pipe; @@ -53,8 +57,16 @@ private: int m_captureIntervalMilliseconds; + int m_depthAlgorithm; double m_averageNumberOfTimes; double m_percentageOfEffectiveArea; + double m_depthRangePercentage; + + double processAveragedImages_roiAvg(const cv::Mat& avgDepthResult, const cv::Mat& mask); + double processAveragedImages_depthRangePercentage(const cv::Mat& avgDepthResult, const cv::Mat& mask); + double processAveragedImages_segmentation(const cv::Mat& avgRgbResult, const cv::Mat& avgDepthResult, const cv::Mat& mask); + + QString getTestFilePath(const QString& fileName); public slots: void OpenDepthCamera(); @@ -101,5 +113,4 @@ signals: private: Ui::DepthCameraClass ui; QThread* m_DepthCameraThread; - }; diff --git a/HPPA/HPPA.cpp b/HPPA/HPPA.cpp index 8ca7fa3..b707fd9 100644 --- a/HPPA/HPPA.cpp +++ b/HPPA/HPPA.cpp @@ -785,7 +785,8 @@ void HPPA::onStartTimedDataCollection(int camType) void HPPA::onObtainTargetDepthInformation(SubTask subTaskParams) { - m_tmc->run4_ObtainTargetDepthInfo(m_depthCameraWindow, subTaskParams.depthType, subTaskParams.depthInfoX, subTaskParams.depthInfoY, subTaskParams.averageNumberOfTimes, subTaskParams.percentageOfEffectiveArea); + m_tmc->run4_ObtainTargetDepthInfo(m_depthCameraWindow, subTaskParams.depthAlgorithm, subTaskParams.depthType, subTaskParams.depthInfoX, subTaskParams.depthInfoY, + subTaskParams.averageNumberOfTimes, subTaskParams.percentageOfEffectiveArea, subTaskParams.depthRangePercentage); } void HPPA::onLiftingPlatform(SubTask subTaskParams) diff --git a/HPPA/TimedDataCollectionDataStructures.cpp b/HPPA/TimedDataCollectionDataStructures.cpp index 9f5be95..fb8f6fe 100644 --- a/HPPA/TimedDataCollectionDataStructures.cpp +++ b/HPPA/TimedDataCollectionDataStructures.cpp @@ -160,11 +160,13 @@ QJsonObject TimedDataCollectionDataStructuresReaderWriter::subTaskToJson(const S obj["autoFocusX"] = subTask.autoFocusX; obj["autoFocusY"] = subTask.autoFocusY; + obj["depthAlgorithm"] = subTask.depthAlgorithm; obj["depthInfoX"] = subTask.depthInfoX; obj["depthInfoY"] = subTask.depthInfoY; obj["averageNumberOfTimes"] = subTask.averageNumberOfTimes; obj["percentageOfEffectiveArea"] = subTask.percentageOfEffectiveArea; obj["depthType"] = subTask.depthType; + obj["depthRangePercentage"] = subTask.depthRangePercentage; return obj; } @@ -187,11 +189,13 @@ bool TimedDataCollectionDataStructuresReaderWriter::jsonToSubTask(const QJsonObj subTask.autoFocusX = json["autoFocusX"].toDouble(); subTask.autoFocusY = json["autoFocusY"].toDouble(); + subTask.depthAlgorithm = json["depthAlgorithm"].toInt(); subTask.depthInfoX = json["depthInfoX"].toDouble(); subTask.depthInfoY = json["depthInfoY"].toDouble(); subTask.averageNumberOfTimes = json["averageNumberOfTimes"].toInt(); subTask.percentageOfEffectiveArea = json["percentageOfEffectiveArea"].toDouble(); subTask.depthType = json["depthType"].toInt(); + subTask.depthRangePercentage = json["depthRangePercentage"].toDouble(); return true; } diff --git a/HPPA/TimedDataCollectionDataStructures.h b/HPPA/TimedDataCollectionDataStructures.h index 4d9be2b..87fb417 100644 --- a/HPPA/TimedDataCollectionDataStructures.h +++ b/HPPA/TimedDataCollectionDataStructures.h @@ -55,11 +55,13 @@ struct SubTask { int captureIntervalSeconds = 5; // 单反/深度相机用 //任务ObtainingDepthInformation所需的x和y坐标 + int depthAlgorithm = 0;//0:深度图像的范围（percentageOfEffectiveArea）平均，1:深度范围（depthRangePercentage）的百分比，2:通过彩色图像分割植被区域的深度图像，然后平均 int depthType = 0;//0表示植被深度，1表示白板/调焦版深度 double depthInfoX = 0.0; double depthInfoY = 0.0; int averageNumberOfTimes = 1; //任务ObtainingDepthInformation所需的平均次数 double percentageOfEffectiveArea = 50.0; //深度图像的有效范围百分比 + double depthRangePercentage = 80.0; //深度范围的百分比 //高光谱自动调焦 HyperImagerType autoFocusHyperImagerType;//取值范围：L、NIR diff --git a/HPPA/TwoMotorControl.cpp b/HPPA/TwoMotorControl.cpp index e722e06..6c7541a 100644 --- a/HPPA/TwoMotorControl.cpp +++ b/HPPA/TwoMotorControl.cpp @@ -210,12 +210,14 @@ void TwoMotorControl::onBack2Origin2() emit back2OriginSignal_TimedDataCollection(); } -void TwoMotorControl::run4_ObtainTargetDepthInfo(DepthCameraWindow* window, int depthType, double depthInfoX, double depthInfoY, int averageNumberOfTimes, double percentageOfEffectiveArea) +void TwoMotorControl::run4_ObtainTargetDepthInfo(DepthCameraWindow* window, double depthAlgorithm,int depthType, double depthInfoX, double depthInfoY, int averageNumberOfTimes, double percentageOfEffectiveArea, double depthRangePercentage) { m_depthType = depthType; + window->m_DepthCameraOperation->setDepthAlgorithm(depthAlgorithm); window->m_DepthCameraOperation->setAverageNumberOfTimes(averageNumberOfTimes); window->m_DepthCameraOperation->setPercentageOfEffectiveArea(percentageOfEffectiveArea); + window->m_DepthCameraOperation->setDepthRangePercentage(depthRangePercentage); m_ObtainTargetDepthInfoCoordinator = new TwoMotor1PosCoordinator(m_multiAxisController); connect(m_ObtainTargetDepthInfoCoordinator, &TwoMotor1PosCoordinator::ArrivalSignal, window, &DepthCameraWindow::OpenDepthCamera_getDepthValue); diff --git a/HPPA/TwoMotorControl.h b/HPPA/TwoMotorControl.h index 355471a..16c120b 100644 --- a/HPPA/TwoMotorControl.h +++ b/HPPA/TwoMotorControl.h @@ -86,7 +86,7 @@ public Q_SLOTS: void run2(SingleLensReflexCameraWindow* w); void run3(DepthCameraWindow* window); - void run4_ObtainTargetDepthInfo(DepthCameraWindow* window, int depthType, double depthInfoX, double depthInfoY, int averageNumberOfTimes, double percentageOfEffectiveArea); + void run4_ObtainTargetDepthInfo(DepthCameraWindow* window, double depthAlgorithm, int depthType, double depthInfoX, double depthInfoY, int averageNumberOfTimes, double percentageOfEffectiveArea, double depthRangePercentage); void run5_AutoFocus(double autoFocusX, double autoFocusY); void onBack2Origin2(); void saveDepthValue(double depthValue); diff --git a/HPPA/about.ui b/HPPA/about.ui index e56d2ec..bc2e921 100644 --- a/HPPA/about.ui +++ b/HPPA/about.ui @@ -288,7 +288,7 @@ QPushButton:pressed } - 版本：3.1.2 + 版本：3.1.3