Mirror
/
EmguCV
mirror of https://github.com/emgucv/emgucv.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

VehicleLicensePlateDetector optimization.

pull/461/head
Canming Huang 4 years ago
parent
308db267ac
commit
7d26341dfa
8 changed files with 133 additions and 141 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 17
      Emgu.CV.Example/XamarinForms/Core/FaceLandmarkDetectionPage.cs
  2. 3
      Emgu.CV.Example/XamarinForms/Core/FindPedestrian.cs
  3. 4
      Emgu.CV.Example/XamarinForms/Core/FreetypePage.cs
  4. 2
      Emgu.CV.Models/Dnn/DetectedObject.cs
  5. 3
      Emgu.CV.Models/Dnn/FaceDetector.cs
  6. 4
      Emgu.CV.Models/Dnn/MaskRcnn.cs
  7. 14
      Emgu.CV.Models/Dnn/MaskedObject.cs
  8. 227
      Emgu.CV.Models/Dnn/VehicleLicensePlateDetector.cs

17
Emgu.CV.Example/XamarinForms/Core/FaceLandmarkDetectionPage.cs
View File

@ -70,26 +70,29 @@ namespace Emgu.CV.XamarinForms
private void DetectAndRender(Mat image)
{
List<Rectangle> fullFaceRegions = new List<Rectangle>();
List<Rectangle> partialFaceRegions = new List<Rectangle>();
List<DetectedObject> fullFaceRegions = new List<DetectedObject>();
List<DetectedObject> partialFaceRegions = new List<DetectedObject>();
_faceDetector.Detect(image, fullFaceRegions, partialFaceRegions);
if (partialFaceRegions.Count > 0)
{
foreach (Rectangle face in partialFaceRegions)
foreach (DetectedObject face in partialFaceRegions)
{
CvInvoke.Rectangle(image, face, new MCvScalar(0, 255, 0));
CvInvoke.Rectangle(image, face.Region, new MCvScalar(0, 255, 0));
}
}
if (fullFaceRegions.Count > 0)
{
foreach (Rectangle face in fullFaceRegions)
foreach (DetectedObject face in fullFaceRegions)
{
CvInvoke.Rectangle(image, face, new MCvScalar(0, 255, 0));
CvInvoke.Rectangle(image, face.Region, new MCvScalar(0, 255, 0));
}
using (VectorOfVectorOfPointF landmarks = _facemarkDetector.Detect(image, fullFaceRegions.ToArray()))
var fullFaceRegionsArr = fullFaceRegions.ToArray();
var rectRegionArr = Array.ConvertAll(fullFaceRegionsArr, r => r.Region);
using (VectorOfVectorOfPointF landmarks = _facemarkDetector.Detect(image, rectRegionArr))
{
int len = landmarks.Size;
for (int i = 0; i < len; i++)

3
Emgu.CV.Example/XamarinForms/Core/FindPedestrian.cs
View File

@ -34,8 +34,6 @@ namespace PedestrianDetection
{
//this is the Cuda version
//watch = Stopwatch.StartNew();
using (GpuMat cudaBgra = new GpuMat())
using (VectorOfRect vr = new VectorOfRect())
@ -50,7 +48,6 @@ namespace PedestrianDetection
{
//this is the CPU/OpenCL version
//watch = Stopwatch.StartNew();
MCvObjectDetection[] results = hog.DetectMultiScale(image);

4
Emgu.CV.Example/XamarinForms/Core/FreetypePage.cs
View File

@ -25,7 +25,6 @@ namespace Emgu.CV.XamarinForms
button.Text = "Draw Freetype Text";
button.Clicked += async (sender, args) =>
{
await InitFreetype();
Mat img = new Mat(new Size(640, 480), DepthType.Cv8U, 3);
img.SetTo(new MCvScalar(0, 0, 0, 0));
@ -47,7 +46,8 @@ namespace Emgu.CV.XamarinForms
manager.AddFile(
"https://github.com/emgucv/emgucv/raw/4.4.0/Emgu.CV.Test/NotoSansCJK-Regular.ttc",
"Freetype");
"Freetype",
"E3C629CB9F416E2E57CFDFEE7573D5CAC3A06A681C105EC081AB9707C1F147E8");
manager.OnDownloadProgressChanged += DownloadManager_OnDownloadProgressChanged;
await manager.Download();

2
Emgu.CV.Models/Dnn/DetectedObject.cs
View File

@ -46,7 +46,7 @@ namespace Emgu.CV.Models
CvInvoke.Rectangle(image, this.Region, color, 2);
CvInvoke.PutText(
image,
String.Format("{0}: {1}", this.Label, this.Confident),
String.Format("{0}: {1}", this.Label == null? this.ClassId.ToString() : this.Label, this.Confident),
this.Region.Location,
FontFace.HersheyDuplex,
1.0,

3
Emgu.CV.Models/Dnn/FaceDetector.cs
View File

@ -16,8 +16,10 @@ using Emgu.CV.Structure;
using Emgu.CV.Util;
using Emgu.Util;
namespace Emgu.CV.Models
{
/// <summary>
/// Face detector using DNN
/// </summary>
@ -61,7 +63,6 @@ namespace Emgu.CV.Models
_faceDetectionModel.SetInputScale(1.0);
_faceDetectionModel.SetInputCrop(false);
if (Emgu.CV.Cuda.CudaInvoke.HasCuda)
{
_faceDetectionModel.SetPreferableBackend(Emgu.CV.Dnn.Backend.Cuda);

4
Emgu.CV.Models/Dnn/MaskRcnn.cs
View File

@ -180,7 +180,9 @@ namespace Emgu.CV.Models
}
}
/// <summary>
/// Release the memory associated with this Mask Rcnn detector.
/// </summary>
protected override void DisposeObject()
{
if (_maskRcnnDetector != null)

14
Emgu.CV.Models/Dnn/MaskedObject.cs
View File

@ -23,7 +23,6 @@ using Rectangle = System.Drawing.Rectangle;
namespace Emgu.CV.Models
{
public class MaskedObject : DetectedObject, IDisposable
{
/// <summary> Track whether Dispose has been called. </summary>
@ -52,14 +51,14 @@ namespace Emgu.CV.Models
/// Draw the detected object on the image
/// </summary>
/// <param name="image">The image to draw on</param>
/// <param name="color">The color used for drawing</param>
public override void Render(Mat image, MCvScalar color)
/// <param name="color">The color used for drawing the mask</param>
public override void Render(Mat image, MCvScalar maskColor)
{
CvInvoke.Rectangle(image, Region, new MCvScalar(0, 0, 0, 0), 1);
CvInvoke.PutText(image, String.Format("{0} ({1})", Label, Confident), Region.Location, FontFace.HersheyComplex, 1.0,
new MCvScalar(0, 0, 255), 2);
DrawMask(image, _mask, Region, color);
DrawMask(image, _mask, Region, maskColor);
}
@ -68,7 +67,9 @@ namespace Emgu.CV.Models
using (Mat maskLarge = new Mat())
using (Mat maskLargeInv = new Mat())
using (Mat subRegion = new Mat(image, rect))
using (Mat largeColor = new Mat(subRegion.Size, Emgu.CV.CvEnum.DepthType.Cv8U,
using (Mat largeColor = new Mat(
subRegion.Size,
Emgu.CV.CvEnum.DepthType.Cv8U,
3))
{
CvInvoke.Resize(mask, maskLarge, rect.Size);
@ -94,7 +95,6 @@ namespace Emgu.CV.Models
CvInvoke.CvtColor(bgrSubRegion, subRegion,
ColorConversion.Bgr2Bgra);
}
}
else
CvInvoke.BlendLinear(largeColor, subRegion, maskLarge, maskLargeInv,
@ -180,6 +180,4 @@ namespace Emgu.CV.Models
}
}
}

227
Emgu.CV.Models/Dnn/VehicleLicensePlateDetector.cs
View File

@ -81,9 +81,12 @@ namespace Emgu.CV.Models
private String _modelFolderName = "vehicle-license-plate-detection-barrier-0106-openvino-2021.2";
private Net _vehicleLicensePlateDetector = null;
private DetectionModel _vehicleLicensePlateDetectionModel = null;
private Net _vehicleAttrRecognizer = null;
private Net _ocr = null;
private async Task InitOCR(System.Net.DownloadProgressChangedEventHandler onDownloadProgressChanged = null)
{
if (_ocr == null)
@ -105,9 +108,30 @@ namespace Emgu.CV.Models
}
await manager.Download();
_ocr =
DnnInvoke.ReadNetFromModelOptimizer(manager.Files[0].LocalFile, manager.Files[1].LocalFile);
using (Mat seqInd = new Mat(
new Size(1, 88),
DepthType.Cv32F,
1))
{
if (seqInd.Depth == DepthType.Cv32F)
{
float[] seqIndValues = new float[seqInd.Width * seqInd.Height];
for (int j = 1; j < seqIndValues.Length; j++)
{
seqIndValues[j] = 1.0f;
}
seqIndValues[0] = 0.0f;
seqInd.SetTo(seqIndValues);
}
_ocr.SetInput(seqInd, "seq_ind");
}
/*
if (Emgu.CV.Cuda.CudaInvoke.HasCuda)
{
@ -132,7 +156,6 @@ namespace Emgu.CV.Models
_modelFolderName,
"492520E55F452223E767D54227D6EF6B60B0C1752DD7B9D747BE65D57B685A0E");
manager.OnDownloadProgressChanged += onDownloadProgressChanged;
await manager.Download();
_vehicleAttrRecognizer =
@ -162,6 +185,15 @@ namespace Emgu.CV.Models
manager.OnDownloadProgressChanged += onDownloadProgressChanged;
await manager.Download();
_vehicleLicensePlateDetectionModel =
new DetectionModel(manager.Files[1].LocalFile, manager.Files[0].LocalFile);
_vehicleLicensePlateDetectionModel.SetInputSize(new Size(300, 300));
_vehicleLicensePlateDetectionModel.SetInputMean(new MCvScalar());
_vehicleLicensePlateDetectionModel.SetInputScale(1.0);
_vehicleLicensePlateDetectionModel.SetInputSwapRB(false);
_vehicleLicensePlateDetectionModel.SetInputCrop(false);
_vehicleLicensePlateDetector =
DnnInvoke.ReadNetFromModelOptimizer(manager.Files[0].LocalFile, manager.Files[1].LocalFile);
@ -213,156 +245,115 @@ namespace Emgu.CV.Models
/// <returns>The detected vehicles.</returns>
public Vehicle[] Detect(Mat image)
{
int imgDim = 300;
int vehicleAttrSize = 72;
MCvScalar meanVal = new MCvScalar();
double scale = 1.0;
float vehicleConfidenceThreshold = 0.5f;
float licensePlateConfidenceThreshold = 0.5f;
//MCvScalar meanVal = new MCvScalar(127.5, 127.5, 127.5);
//double scale = 127.5;
Size imageSize = image.Size;
using (Mat inputBlob = DnnInvoke.BlobFromImage(
image,
scale,
new Size(imgDim, imgDim),
meanVal,
false,
false,
DepthType.Cv32F))
_vehicleLicensePlateDetector.SetInput(inputBlob, "Placeholder");
int vehicleAttrSize = 72;
double scale = 1.0;
MCvScalar meanVal = new MCvScalar();
List<Vehicle> vehicles = new List<Vehicle>();
List<LicensePlate> plates = new List<LicensePlate>();
using (Mat detection = _vehicleLicensePlateDetector.Forward("DetectionOutput_"))
foreach (DetectedObject vehicleOrPlate in _vehicleLicensePlateDetectionModel.Detect(image, 0.0f, 0.0f))
{
float[,,,] values = detection.GetData(true) as float[,,,];
for (int i = 0; i < values.GetLength(2); i++)
Rectangle region = vehicleOrPlate.Region;
if (vehicleOrPlate.ClassId == 1 && vehicleOrPlate.Confident > vehicleConfidenceThreshold)
{
float imageId = values[0, 0, i, 0];
float label = values[0, 0, i, 1]; //if label == 1, it is a vehicle; if label == 2, it is a license plate
float confident = values[0, 0, i, 2];
Rectangle region = DetectedObject.GetRectangle(
values[0, 0, i, 3],
values[0, 0, i, 4],
values[0, 0, i, 5],
values[0, 0, i, 6],
imageSize.Width,
imageSize.Height);
if (label == 1 && confident > vehicleConfidenceThreshold)
{ //this is a vehicle
Vehicle v = new Vehicle();
v.Region = region;
#region find out the type and color of the vehicle
using (Mat vehicle = new Mat(image, region))
//this is a vehicle
Vehicle v = new Vehicle();
v.Region = region;
#region find out the type and color of the vehicle
using (Mat vehicle = new Mat(image, region))
{
using (Mat vehicleBlob = DnnInvoke.BlobFromImage(
vehicle,
scale,
new Size(vehicleAttrSize, vehicleAttrSize),
meanVal,
false,
false,
DepthType.Cv32F))
{
using (Mat vehicleBlob = DnnInvoke.BlobFromImage(
vehicle,
scale,
new Size(vehicleAttrSize, vehicleAttrSize),
meanVal,
false,
false,
DepthType.Cv32F))
{
_vehicleAttrRecognizer.SetInput(vehicleBlob, "input");
_vehicleAttrRecognizer.SetInput(vehicleBlob, "input");
using (VectorOfMat vm = new VectorOfMat(2))
using (VectorOfMat vm = new VectorOfMat(2))
{
_vehicleAttrRecognizer.Forward(vm, new string[] { "color", "type" });
using (Mat vehicleColorMat = vm[0])
using (Mat vehicleTypeMat = vm[1])
{
_vehicleAttrRecognizer.Forward(vm, new string[] { "color", "type" });
using (Mat vehicleColorMat = vm[0])
using (Mat vehicleTypeMat = vm[1])
{
float[] vehicleColorData = vehicleColorMat.GetData(false) as float[];
float maxProbColor = vehicleColorData.Max();
int maxIdxColor = Array.IndexOf(vehicleColorData, maxProbColor);
v.Color = _colorName[maxIdxColor];
float[] vehicleTypeData = vehicleTypeMat.GetData(false) as float[];
float maxProbType = vehicleTypeData.Max();
int maxIdxType = Array.IndexOf(vehicleTypeData, maxProbType);
v.Type = _vehicleType[maxIdxType];
}
float[] vehicleColorData = vehicleColorMat.GetData(false) as float[];
float maxProbColor = vehicleColorData.Max();
int maxIdxColor = Array.IndexOf(vehicleColorData, maxProbColor);
v.Color = _colorName[maxIdxColor];
float[] vehicleTypeData = vehicleTypeMat.GetData(false) as float[];
float maxProbType = vehicleTypeData.Max();
int maxIdxType = Array.IndexOf(vehicleTypeData, maxProbType);
v.Type = _vehicleType[maxIdxType];
}
}
}
#endregion
vehicles.Add(v);
}
#endregion
if (label == 2 && confident > licensePlateConfidenceThreshold)
{ //this is a license plate
LicensePlate p = new LicensePlate();
p.Region = region;
vehicles.Add(v);
}
else if (vehicleOrPlate.ClassId == 2 && vehicleOrPlate.Confident > licensePlateConfidenceThreshold)
{
//this is a license plate
LicensePlate p = new LicensePlate();
p.Region = region;
#region OCR on license plate
using (Mat plate = new Mat(image, region))
#region OCR on license plate
using (Mat plate = new Mat(image, region))
{
using (Mat inputBlob = DnnInvoke.BlobFromImage(
plate,
scale,
new Size(94, 24),
meanVal,
false,
false,
DepthType.Cv32F))
{
using (Mat inputBlob = DnnInvoke.BlobFromImage(
plate,
scale,
new Size(94, 24),
meanVal,
false,
false,
DepthType.Cv32F))
using (Mat seqInd = new Mat(
new Size(1, 88),
DepthType.Cv32F,
1))
_ocr.SetInput(inputBlob, "data");
using (Mat output = _ocr.Forward("decode"))
{
_ocr.SetInput(inputBlob, "data");
if (seqInd.Depth == DepthType.Cv32F)
float[] plateValue = output.GetData(false) as float[];
StringBuilder licensePlateStringBuilder = new StringBuilder();
foreach (int j in plateValue)
{
float[] seqIndValues = new float[seqInd.Width * seqInd.Height];
for (int j = 1; j < seqIndValues.Length; j++)
if (j >= 0)
{
seqIndValues[j] = 1.0f;
licensePlateStringBuilder.Append(_plateText[j]);
}
seqIndValues[0] = 0.0f;
seqInd.SetTo(seqIndValues);
}
_ocr.SetInput(seqInd, "seq_ind");
using (Mat output = _ocr.Forward("decode"))
{
float[] plateValue = output.GetData(false) as float[];
StringBuilder licensePlateStringBuilder = new StringBuilder();
foreach (int j in plateValue)
{
if (j >= 0)
{
licensePlateStringBuilder.Append(_plateText[j]);
}
}
p.Text = licensePlateStringBuilder.ToString();
}
p.Text = licensePlateStringBuilder.ToString();
}
}
#endregion
plates.Add(p);
}
#endregion
plates.Add(p);
}
}
foreach (LicensePlate p in plates)
foreach (LicensePlate p in plates)
{
foreach (Vehicle v in vehicles)
{
foreach (Vehicle v in vehicles)
if (v.ContainsPlate(p))
{
if (v.ContainsPlate(p))
{
v.LicensePlate = p;
break;
}
v.LicensePlate = p;
break;
}
}
}
return vehicles.ToArray();
}

Write Preview
Loading…
Cancel
Save