From 8b2319fe8107f757f446bab3ff096112fc3e3903 Mon Sep 17 00:00:00 2001 From: DucDangAnh Date: Thu, 18 Jun 2020 16:14:21 +0700 Subject: [PATCH] Add class PlateProcessor. Update Event PlateCamera_OnOneVideoFrameRequested. PlateDetector add method: ExtendPlateRect --- .../AIParkingApplication.csproj | 1 + AIParkingApplication/LaneIn.cs | 41 ++----- AIParkingApplication/PlateDetector.cs | 24 ++++- AIParkingApplication/PlateProcessor.cs | 101 ++++++++++++++++++ 4 files changed, 134 insertions(+), 33 deletions(-) create mode 100644 AIParkingApplication/PlateProcessor.cs diff --git a/AIParkingApplication/AIParkingApplication.csproj b/AIParkingApplication/AIParkingApplication.csproj index 345aee6..d70788e 100644 --- a/AIParkingApplication/AIParkingApplication.csproj +++ b/AIParkingApplication/AIParkingApplication.csproj @@ -92,6 +92,7 @@ LaneOut.cs + diff --git a/AIParkingApplication/LaneIn.cs b/AIParkingApplication/LaneIn.cs index 9a35cce..85d33a8 100644 --- a/AIParkingApplication/LaneIn.cs +++ b/AIParkingApplication/LaneIn.cs @@ -1,8 +1,6 @@ using OpenCvSharp; using OpenCvSharp.Extensions; using System; -using System.Drawing; -using System.IO; using System.Threading.Tasks; using System.Windows.Forms; @@ -12,18 +10,20 @@ namespace AIParkingApplication { private Camera overviewCamera; private Camera plateCamera; - private PlateDetector squarePlateDetector; + private PlateProcessor plateProcessor; public LaneIn() { InitializeComponent(); string overviewCameraVideo = @"rtsp://wowzaec2demo.streamlock.net/vod/mp4:BigBuckBunny_115k.mov"; //@"C:\HS_test.mp4" string plateCameraVideo = @"C:\HS_test.mp4"; + //string plateCameraVideo = @"C:\CongRa_1.mp4"; + overviewCamera = new Camera(overviewCameraVideo); plateCamera = new Camera(plateCameraVideo); plateCamera.OnVideoFrameReceived += PlateCameraOnVideoFrameReceived; - plateCamera.OnOneVideoFrameRequested += PlateCamera_OnOneVideoFrameRequested; ; + plateCamera.OnOneVideoFrameRequested += PlateCamera_OnOneVideoFrameRequested; overviewCamera.OnVideoFrameReceived += OverviewCameraOnVideoFrameReceived; overviewCamera.OnOneVideoFrameRequested += OverviewCamera_OnOneVideoFrameRequested; @@ -31,7 +31,7 @@ namespace AIParkingApplication overviewCamera.Startcapture(); plateCamera.Startcapture(); - squarePlateDetector = new PlateDetector(PlateType.Square, PlateDetectorConstant.MIN_SIZE_DEFAULT_SQUARE_PLATE, PlateDetectorConstant.MAX_SIZE_DEFAULT_SQUARE_PLATE, PlateDetectorConstant.SCALE_FACTOR_DEFAULT_SQUARE_PLATE, PlateDetectorConstant.MIN_NEIGHBORS_DEFAULT_SQUARE_PLATE); + plateProcessor = new PlateProcessor(true, false); } private void OverviewCamera_OnOneVideoFrameRequested(Mat videoFrame) @@ -47,38 +47,17 @@ namespace AIParkingApplication { Task.Factory.StartNew(new Action(async () => { - //TODO: check size before resizing - Cv2.Resize(videoFrame, videoFrame, new OpenCvSharp.Size(1280, 720)); - Mat result = squarePlateDetector.DetectPlate(videoFrame); - - //TODO: check size before resizing - Cv2.Resize(result, result, new OpenCvSharp.Size(272, 272)); - OcrResult ocrResult = await Util.SendEngineRequestAsync(result, PlateType.Square); - - //TODO: Exception Here - Bitmap resultPlateImage; - if (!string.IsNullOrEmpty(ocrResult.Ocr) && !string.IsNullOrEmpty(ocrResult.Plate)) - { - var imageData = Convert.FromBase64String(ocrResult.Plate); - using (var ms = new MemoryStream(imageData)) - { - resultPlateImage = new Bitmap(ms); - } - } - else - { - resultPlateImage = result.ToBitmap(); - } + var finalPlateResult = await plateProcessor.ProcessPlate(videoFrame); lblPlateString.Invoke(new Action(() => { - lblPlateString.Text = ocrResult.Ocr; + lblPlateString.Text = finalPlateResult.PlateString; })); pictureBoxPlateImage.Invoke(new Action(() => { pictureBoxPlateImage.Image?.Dispose(); - pictureBoxPlateImage.Image = resultPlateImage; + pictureBoxPlateImage.Image = finalPlateResult.PlateImage; })); })); } @@ -103,8 +82,8 @@ namespace AIParkingApplication private void CaptureAllCamera() { - this.plateCamera.RequestCaptureOneFrame(); - this.overviewCamera.RequestCaptureOneFrame(); + plateCamera.RequestCaptureOneFrame(); + overviewCamera.RequestCaptureOneFrame(); } private void button1_Click(object sender, EventArgs e) diff --git a/AIParkingApplication/PlateDetector.cs b/AIParkingApplication/PlateDetector.cs index dbcfd40..8d44e69 100644 --- a/AIParkingApplication/PlateDetector.cs +++ b/AIParkingApplication/PlateDetector.cs @@ -38,11 +38,31 @@ namespace AIParkingApplication } } + private Rect ExtendPlateRect(Rect plateRect, int extendWidth = 50, int extendHeigh = 50) //TODO: 50 for plate square. + { + int newX = plateRect.X - extendWidth; + int newY = plateRect.Y - extendHeigh; + int newWidth = plateRect.Width + extendWidth * 2; + int newHeigh = plateRect.Height + extendHeigh * 2; + return new Rect(newX, newY, newWidth, newHeigh); + } + private Mat GetBiggestPlate(Rect[] plateRects, Mat frame) { Rect biggestPlateRect = plateRects.OrderByDescending(x => x.Width).FirstOrDefault(); + Console.WriteLine("==================================================="); + for (int i = 0; i < plateRects.Length; i++) + { + Console.WriteLine($"{i}\t{plateRects[i].Width}\t{plateRects[i].Height}"); + } Console.WriteLine($"GetBiggestPlate - Width: {biggestPlateRect.Width}\tHeight: {biggestPlateRect.Height}"); - Mat plateImage = frame[biggestPlateRect]; + + Rect extendedRect = ExtendPlateRect(biggestPlateRect); + //TODO: check oversize frame + Mat plateImage = frame[extendedRect]; + //Cv2.ImShow("Old", frame[biggestPlateRect]); + //Cv2.ImShow("New", frame[extendedRect]); + //Cv2.WaitKey(); return plateImage; } } @@ -57,7 +77,7 @@ namespace AIParkingApplication { public const double SCALE_FACTOR_DEFAULT_SQUARE_PLATE = 1.03; public static int MIN_NEIGHBORS_DEFAULT_SQUARE_PLATE = 2; - public static Size MIN_SIZE_DEFAULT_SQUARE_PLATE = new Size(80, 75); + public static Size MIN_SIZE_DEFAULT_SQUARE_PLATE = new Size(105, 100); public static Size MAX_SIZE_DEFAULT_SQUARE_PLATE = new Size(450, 400); public static double SCALE_FACTOR_DEFAULT_LONG_PLATE = 1.5; diff --git a/AIParkingApplication/PlateProcessor.cs b/AIParkingApplication/PlateProcessor.cs new file mode 100644 index 0000000..ded2f77 --- /dev/null +++ b/AIParkingApplication/PlateProcessor.cs @@ -0,0 +1,101 @@ +using OpenCvSharp; +using OpenCvSharp.Extensions; +using System; +using System.Drawing; +using System.IO; +using System.Threading.Tasks; + +namespace AIParkingApplication +{ + public class PlateProcessor + { + private bool isSupportSquarePlate; + private bool isSupportLongPlate; + private PlateDetector squarePlateDetector; + private PlateDetector longPlateDetector; + + public PlateProcessor(bool isSupportSquarePlate = true, bool isSupportLongPlate = false) + { + this.isSupportSquarePlate = isSupportSquarePlate; + this.isSupportLongPlate = isSupportLongPlate; + + if (this.isSupportSquarePlate) + { + squarePlateDetector = new PlateDetector(PlateType.Square, PlateDetectorConstant.MIN_SIZE_DEFAULT_SQUARE_PLATE, PlateDetectorConstant.MAX_SIZE_DEFAULT_SQUARE_PLATE, PlateDetectorConstant.SCALE_FACTOR_DEFAULT_SQUARE_PLATE, PlateDetectorConstant.MIN_NEIGHBORS_DEFAULT_SQUARE_PLATE); + } + + if (this.isSupportLongPlate) + { + longPlateDetector = new PlateDetector(PlateType.Long, PlateDetectorConstant.MIN_SIZE_DEFAULT_LONG_PLATE, PlateDetectorConstant.MAX_SIZE_DEFAULT_LONG_PLATE, PlateDetectorConstant.SCALE_FACTOR_DEFAULT_LONG_PLATE, PlateDetectorConstant.MIN_NEIGHBORS_DEFAULT_LONG_PLATE); + } + } + + private async Task DetectPlateAndDoOcrEngineAsync(PlateType plateType, Mat frame) + { + //TODO: check size before resizing + //Cv2.Resize(frame, frame, new OpenCvSharp.Size(1280, 720)); + Mat plateDetected = plateType == PlateType.Square ? squarePlateDetector.DetectPlate(frame) : longPlateDetector.DetectPlate(frame); + //Cv2.ImShow("dasdas", plateDetected); + //Cv2.WaitKey(1); + //if (plateType == PlateType.Square) + //{ + // //TODO: check size before resizing + // Cv2.Resize(plateDetected, plateDetected, new OpenCvSharp.Size(272, 272)); + //} + + //TODO: check size before resizing + Cv2.Resize(plateDetected, plateDetected, new OpenCvSharp.Size(272, 272)); + + //TODO: Check if plateDetected empty + OcrResult plateOcrResultFromEngine = await Util.SendEngineRequestAsync(plateDetected, plateType); + Bitmap finalPlateImage; + if (!string.IsNullOrEmpty(plateOcrResultFromEngine.Ocr) && !string.IsNullOrEmpty(plateOcrResultFromEngine.Plate)) + { + byte[] imageData = Convert.FromBase64String(plateOcrResultFromEngine.Plate); + using (var ms = new MemoryStream(imageData)) + { + finalPlateImage = new Bitmap(ms); + } + } + else + { + finalPlateImage = plateDetected.ToBitmap(); + } + + return new FinalPlateResult + { + PlateString = plateOcrResultFromEngine.Ocr, + PlateImage = finalPlateImage + }; + } + + public async Task ProcessPlate(Mat frame) + { + //TODO: check size before resizing + Cv2.Resize(frame, frame, new OpenCvSharp.Size(1280, 720)); + FinalPlateResult plateResult; + + if (isSupportLongPlate) + { + plateResult = await DetectPlateAndDoOcrEngineAsync(PlateType.Long, frame); + if (string.IsNullOrEmpty(plateResult.PlateString)) + { + plateResult = await DetectPlateAndDoOcrEngineAsync(PlateType.Square, frame); + } + } + else + { + Console.WriteLine("Recall Square Plates"); + plateResult = await DetectPlateAndDoOcrEngineAsync(PlateType.Square, frame); + } + + return plateResult; + } + + public class FinalPlateResult + { + public string PlateString { get; set; } + public Bitmap PlateImage { get; set; } + } + } +}