diff --git a/Test_Merge/App.config b/Test_Merge/App.config index 56efbc7..7d84477 100644 --- a/Test_Merge/App.config +++ b/Test_Merge/App.config @@ -1,6 +1,14 @@ - + + + + + + + + + \ No newline at end of file diff --git a/Test_Merge/DriverData.cs b/Test_Merge/DriverData.cs new file mode 100644 index 0000000..8f89f5b --- /dev/null +++ b/Test_Merge/DriverData.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Test_Merge +{ + public class DriverData + { + public bool DRS; //True = Drs is opened + public int GapToLeader; //In ms + public int LapTime; //In ms + public string Name; //Ex: LECLERC + public int Position; //Ex: 1 + public int Sector1; //in ms + public int Sector2; //in ms + public int Sector3; //in ms + public Tyre CurrentTyre;//Ex Soft 11 laps + + public DriverData(bool dRS, int gapToLeader, int lapTime, string name, int position, int sector1, int sector2, int sector3, Tyre tyre) + { + DRS = dRS; + GapToLeader = gapToLeader; + LapTime = lapTime; + Name = name; + Position = position; + Sector1 = sector1; + Sector2 = sector2; + Sector3 = sector3; + CurrentTyre = tyre; + } + public DriverData() + { + DRS = false; + GapToLeader = -1; + LapTime = -1; + Name = "Unknown"; + Position = -1; + Sector1 = -1; + Sector2 = -1; + Sector3 = -1; + CurrentTyre = new Tyre(Tyre.Type.Undefined, -1); + } + /// + /// Method that displays all the data found in a string + /// + /// string containing all the driver datas + public override string ToString() + { + string result = ""; + + //Position + result += "Position : " + Position + Environment.NewLine; + //Gap + result += "Gap to leader : " + Reader.ConvertMsToTime(GapToLeader) + Environment.NewLine; + //LapTime + result += "Lap time : " + Reader.ConvertMsToTime(LapTime) + Environment.NewLine; + //DRS + result += "DRS : " + DRS + Environment.NewLine; + //Tyres + result += "Uses " + CurrentTyre.Coumpound + " tyre " + CurrentTyre.NumberOfLaps + " laps old" + Environment.NewLine; + //Name + result += "Driver name : " + Name + Environment.NewLine; + //Sector 1 + result += "Sector 1 : " + Reader.ConvertMsToTime(Sector1) + Environment.NewLine; + //Sector 1 + result += "Sector 2 : " + Reader.ConvertMsToTime(Sector2) + Environment.NewLine; + //Sector 1 + result += "Sector 3 : " + Reader.ConvertMsToTime(Sector3) + Environment.NewLine; + + return result; + } + } + //Structure to store tyres infos + public struct Tyre + { + //If new tyres were to be added you will have to need to change this enum + public enum Type + { + Soft, + Medium, + Hard, + Inter, + Wet, + Undefined + } + public Type Coumpound; + public int NumberOfLaps; + public Tyre(Type type, int laps) + { + Coumpound = type; + NumberOfLaps = laps; + } + } +} diff --git a/Test_Merge/DriverDrsWindow.cs b/Test_Merge/DriverDrsWindow.cs new file mode 100644 index 0000000..94a30d8 --- /dev/null +++ b/Test_Merge/DriverDrsWindow.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Test_Merge +{ + internal class DriverDrsWindow:Window + { + private static int EmptyDrsGreenValue = -1; + private static Random rnd = new Random(); + public DriverDrsWindow(Bitmap image, Rectangle bounds) : base(image, bounds) + { + Name = "DRS"; + } + public override async Task DecodePng() + { + bool result = false; + int greenValue = GetGreenPixels(); + if (EmptyDrsGreenValue == -1) + EmptyDrsGreenValue = greenValue; + + if (greenValue > EmptyDrsGreenValue + EmptyDrsGreenValue / 100 * 30) + result = true; + + return result; + } + private unsafe int GetGreenPixels() + { + int tot = 0; + + Bitmap bmp = WindowImage; + Rectangle rect = new Rectangle(0, 0, bmp.Width, bmp.Height); + BitmapData bmpData = bmp.LockBits(rect, ImageLockMode.ReadOnly, bmp.PixelFormat); + int bytesPerPixel = Bitmap.GetPixelFormatSize(bmp.PixelFormat) / 8; + + unsafe + { + byte* ptr = (byte*)bmpData.Scan0.ToPointer(); + for (int y = 0; y < bmp.Height; y++) + { + byte* currentLine = ptr + (y * bmpData.Stride); + for (int x = 0; x < bmp.Width; x++) + { + byte* pixel = currentLine + (x * bytesPerPixel); + + byte blue = pixel[0]; + byte green = pixel[1]; + byte red = pixel[2]; + + if (green > blue * 1.5 && green > red * 1.5) + { + tot++; + } + } + } + } + bmp.UnlockBits(bmpData); + + return tot; + } + public Rectangle GetBox() + { + var tessImage = Pix.LoadFromMemory(ImageToByte(WindowImage)); + Engine.SetVariable("tessedit_char_whitelist", ""); + Page page = Engine.Process(tessImage); + + using (var iter = page.GetIterator()) + { + iter.Begin(); + do + { + Rect boundingBox; + + // Get the bounding box for the current element + if (iter.TryGetBoundingBox(PageIteratorLevel.Word, out boundingBox)) + { + page.Dispose(); + return new Rectangle(boundingBox.X1, boundingBox.X2, boundingBox.Width, boundingBox.Height); + } + } while (iter.Next(PageIteratorLevel.Word)); + + page.Dispose(); + return new Rectangle(0, 0, 0, 0); + } + } + } +} diff --git a/Test_Merge/DriverGapToLeaderWindow.cs b/Test_Merge/DriverGapToLeaderWindow.cs new file mode 100644 index 0000000..bbd79db --- /dev/null +++ b/Test_Merge/DriverGapToLeaderWindow.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Test_Merge +{ + internal class DriverGapToLeaderWindow:Window + { + public DriverGapToLeaderWindow(Bitmap image, Rectangle bounds) : base(image, bounds) + { + Name = "Gap to leader"; + } + /// + /// Decodes the gap to leader using Tesseract OCR + /// + /// + public override async Task DecodePng() + { + int result = await GetTimeFromPng(WindowImage, OcrImage.WindowType.Gap, Engine); + return result; + } + } +} diff --git a/Test_Merge/DriverLapTimeWindow.cs b/Test_Merge/DriverLapTimeWindow.cs new file mode 100644 index 0000000..4bc5d7a --- /dev/null +++ b/Test_Merge/DriverLapTimeWindow.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Drawing; + +namespace Test_Merge +{ + internal class DriverLapTimeWindow:Window + { + public DriverLapTimeWindow(Bitmap image, Rectangle bounds) : base(image, bounds) + { + Name = "Lap time"; + } + /// + /// Decodes the lap time contained in the image using OCR Tesseract + /// + /// The laptime in int (ms) + public override async Task DecodePng() + { + int result = await GetTimeFromPng(WindowImage, OcrImage.WindowType.LapTime, Engine); + return result; + } + } +} diff --git a/Test_Merge/DriverNameWindow.cs b/Test_Merge/DriverNameWindow.cs new file mode 100644 index 0000000..ca08a2a --- /dev/null +++ b/Test_Merge/DriverNameWindow.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Drawing; + +namespace Test_Merge +{ + public class DriverNameWindow : Window + { + public static Random rnd = new Random(); + public DriverNameWindow(Bitmap image, Rectangle bounds) : base(image, bounds) + { + Name = "Name"; + } + /// + /// Decodes using OCR wich driver name is in the image + /// + /// + /// The driver name in string + public override async Task DecodePng(List DriverList) + { + string result = ""; + result = await GetStringFromPng(WindowImage, Engine); + + if (!IsADriver(DriverList, result)) + { + //I put everything in uppercase to try to lower the chances of bad answers + result = FindClosestMatch(DriverList.ConvertAll(d => d.ToUpper()), result.ToUpper()); + } + return result; + } + /// + /// Verifies that the name found in the OCR is a valid name + /// + /// + /// + /// If ye or no the driver exists + private static bool IsADriver(List driverList, string potentialDriver) + { + bool result = false; + //I cant use drivers.Contains because it has missmatched cases and all + foreach (string name in driverList) + { + if (name.ToUpper() == potentialDriver.ToUpper()) + result = true; + } + return result; + } + } +} diff --git a/Test_Merge/DriverPositionWindow.cs b/Test_Merge/DriverPositionWindow.cs new file mode 100644 index 0000000..6fe3f6d --- /dev/null +++ b/Test_Merge/DriverPositionWindow.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Drawing; + +namespace Test_Merge +{ + public class DriverPositionWindow:Window + { + public DriverPositionWindow(Bitmap image, Rectangle bounds) : base(image, bounds) + { + Name = "Position"; + } + /// + /// Decodes the position number using Tesseract OCR + /// + /// The position of the pilot in int + public override async Task DecodePng() + { + string ocrResult = await GetStringFromPng(WindowImage, Engine, "0123456789"); + + int position; + try + { + position = Convert.ToInt32(ocrResult); + } + catch + { + position = -1; + } + return position; + } + } +} diff --git a/Test_Merge/DriverSectorWindow.cs b/Test_Merge/DriverSectorWindow.cs new file mode 100644 index 0000000..c08d845 --- /dev/null +++ b/Test_Merge/DriverSectorWindow.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Drawing; + +namespace Test_Merge +{ + internal class DriverSectorWindow:Window + { + public DriverSectorWindow(Bitmap image, Rectangle bounds) : base(image, bounds) + { + Name = "Sector 1"; + } + /// + /// Decodes the sector + /// + /// the sector time in int (ms) + public override async Task DecodePng() + { + int ocrResult = await GetTimeFromPng(WindowImage, OcrImage.WindowType.Sector, Engine); + return ocrResult; + } + } +} diff --git a/Test_Merge/DriverTyresWindow.cs b/Test_Merge/DriverTyresWindow.cs new file mode 100644 index 0000000..213459c --- /dev/null +++ b/Test_Merge/DriverTyresWindow.cs @@ -0,0 +1,135 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Drawing; + +namespace Test_Merge +{ + public class DriverTyresWindow:Window + { + private static Random rnd = new Random(); + int seed = rnd.Next(0, 10000); + + //Those are the colors I found but you can change them if they change in the future like in 2019 + public static Color SOFT_TYRE_COLOR = Color.FromArgb(0xff, 0x00, 0x00); + public static Color MEDIUM_TYRE_COLOR = Color.FromArgb(0xf5, 0xbf, 0x00); + public static Color HARD_TYRE_COLOR = Color.FromArgb(0xa4, 0xa5, 0xa8); + public static Color INTER_TYRE_COLOR = Color.FromArgb(0x00, 0xa4, 0x2e); + public static Color WET_TYRE_COLOR = Color.FromArgb(0x27, 0x60, 0xa6); + public static Color EMPTY_COLOR = Color.FromArgb(0x20, 0x20, 0x20); + + public DriverTyresWindow(Bitmap image, Rectangle bounds) : base(image, bounds) + { + Name = "Tyres"; + } + /// + /// This will decode the content of the image + /// + /// And object containing what was on the image + public override async Task DecodePng() + { + return await GetTyreInfos(); + } + /// + /// Method that will decode whats on the image and return the tyre infos it could manage to recover + /// + /// A tyre object containing tyre infos + private async Task GetTyreInfos() + { + Bitmap tyreZone = GetSmallBitmapFromBigOne(WindowImage, FindTyreZone()); + Tyre.Type type = Tyre.Type.Undefined; + type = GetTyreTypeFromColor(OcrImage.GetAvgColorFromBitmap(tyreZone)); + int laps = -1; + + string number = await GetStringFromPng(tyreZone, Engine, "0123456789", OcrImage.WindowType.Tyre); + try + { + laps = Convert.ToInt32(number); + } + catch + { + //We could not convert the number so its a letter so its 0 laps old + laps = 0; + } + //tyreZone.Save(Reader.DEBUG_DUMP_FOLDER + "Tyre" + type + "Laps" + laps + '#' + rnd.Next(0, 1000) + ".png"); + return new Tyre(type, laps); + } + /// + /// Finds where the important part of the image is + /// + /// A rectangle containing position and dimensions of the important part of the image + private Rectangle FindTyreZone() + { + Bitmap bmp = WindowImage; + int currentPosition = bmp.Width; + int height = bmp.Height / 2; + Color limitColor = Color.FromArgb(0x50, 0x50, 0x50); + Color currentColor = Color.FromArgb(0, 0, 0); + + Size newWindowSize = new Size(bmp.Height - Convert.ToInt32((float)bmp.Height / 100f * 25f), bmp.Height - Convert.ToInt32((float)bmp.Height / 100f * 35f)); + + while (currentColor.R <= limitColor.R && currentColor.G <= limitColor.G && currentColor.B <= limitColor.B && currentPosition > 0) + { + currentPosition--; + currentColor = bmp.GetPixel(currentPosition, height); + } + + //Its here to let the new window include a little bit of the right + int CorrectedX = currentPosition - (newWindowSize.Width) + Convert.ToInt32((float)newWindowSize.Width / 100f * 10f); + int CorrectedY = Convert.ToInt32((float)newWindowSize.Height / 100f * 35f); + if (CorrectedX <= 0) + return new Rectangle(0, 0, newWindowSize.Width, newWindowSize.Height); + + return new Rectangle(CorrectedX, CorrectedY, newWindowSize.Width, newWindowSize.Height); + } + //This method has been created with the help of chatGPT + /// + /// Methods that compares a list of colors to see wich is the closest from the input color and decide wich tyre type it is + /// + /// The color that you found + /// The tyre type + public Tyre.Type GetTyreTypeFromColor(Color inputColor) + { + Tyre.Type type = Tyre.Type.Undefined; + List colors = new List(); + //dont forget that if for some reason someday F1 adds a new Tyre type you will need to add it in the constants but also here in the list + //You will also need to add it below in the Tyre object's enum and add an if in the end of this method + colors.Add(SOFT_TYRE_COLOR); + colors.Add(MEDIUM_TYRE_COLOR); + colors.Add(HARD_TYRE_COLOR); + colors.Add(INTER_TYRE_COLOR); + colors.Add(WET_TYRE_COLOR); + colors.Add(EMPTY_COLOR); + + Color closestColor = colors[0]; + int closestDistance = int.MaxValue; + foreach (Color color in colors) + { + int distance = Math.Abs(color.R - inputColor.R) + Math.Abs(color.G - inputColor.G) + Math.Abs(color.B - inputColor.B); + if (distance < closestDistance) + { + closestColor = color; + closestDistance = distance; + } + } + + //We cant use a switch as the colors cant be constants ... + if (closestColor == SOFT_TYRE_COLOR) + type = Tyre.Type.Soft; + if (closestColor == MEDIUM_TYRE_COLOR) + type = Tyre.Type.Medium; + if (closestColor == HARD_TYRE_COLOR) + type = Tyre.Type.Hard; + if (closestColor == INTER_TYRE_COLOR) + type = Tyre.Type.Inter; + if (closestColor == WET_TYRE_COLOR) + type = Tyre.Type.Wet; + if (closestColor == EMPTY_COLOR) + return Tyre.Type.Undefined; + + return type; + } + } +} diff --git a/Test_Merge/F1TVEmulator.cs b/Test_Merge/F1TVEmulator.cs index 7a71ef0..ec2854b 100644 --- a/Test_Merge/F1TVEmulator.cs +++ b/Test_Merge/F1TVEmulator.cs @@ -78,9 +78,17 @@ namespace Test_Merge options.AddArgument("--no-startup-window"); //ACTUAL STARTUP + try + { + Driver = new FirefoxDriver(service, options); + } + catch + { + //Could not start the driver, could be because an other instance is running. make sure its closed before re attempting + return 101; + } + - Driver = new FirefoxDriver(service, options); - var loginCookie = new Cookie(loginCookieName, loginCookieValue, COOKIE_HOST, "/", DateTime.Now.AddDays(5)); var loginSessionCookie = new Cookie(loginSessionCookieName, loginSessionValue, COOKIE_HOST, "/", DateTime.Now.AddDays(5)); @@ -91,14 +99,32 @@ namespace Test_Merge Driver.Manage().Cookies.AddCookie(loginCookie); Driver.Manage().Cookies.AddCookie(loginSessionCookie); - Driver.Navigate().GoToUrl("https://f1tv.formula1.com/detail/1000006436/2023-saudi-arabian-grand-prix?action=play"); + try + { + Driver.Navigate().GoToUrl(GrandPrixUrl); + } + catch + { + //The url is not a valid url + Driver.Dispose(); + return 103; + } //Waits for the page to fully load Driver.Manage().Timeouts().PageLoad = TimeSpan.FromSeconds(30); //Removes the cookie prompt - IWebElement conscentButton = Driver.FindElement(By.Id("truste-consent-button")); - conscentButton.Click(); + try + { + IWebElement conscentButton = Driver.FindElement(By.Id("truste-consent-button")); + conscentButton.Click(); + } + catch + { + Driver.Dispose(); + return 104; + } + //Again waits for the page to fully load (when you accept cookies it takes a little time for the page to load) //Cannot use The timeout because the feed loading is not really loading so there is not event or anything @@ -110,12 +136,11 @@ namespace Test_Merge IWebElement dataChannelButton = Driver.FindElement(By.ClassName("data-button")); dataChannelButton.Click(); } - catch (OpenQA.Selenium.NoSuchElementException error) + catch { //If the data button does not exists its because the user is not connected - MessageBox.Show("Could not connect to the F1TV. Please connect to it using your account on chrome and if you already did please wait the process can take a few minuts"); Driver.Dispose(); - return 1; + return 102; } //Open settings // Press the space key, this should make the setting button visible @@ -146,7 +171,7 @@ namespace Test_Merge } public Bitmap Screenshot() { - Bitmap result = new Bitmap(100,100); + Bitmap result = new Bitmap(100, 100); if (Ready) { Screenshot scrsht = ((ITakesScreenshot)Driver).GetScreenshot(); diff --git a/Test_Merge/OCRDecoder.cs b/Test_Merge/OCRDecoder.cs new file mode 100644 index 0000000..b688cd0 --- /dev/null +++ b/Test_Merge/OCRDecoder.cs @@ -0,0 +1,278 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using System.Drawing; +using System.IO; +using System.Text.Json; +using System.Windows.Forms; + +namespace Test_Merge +{ + internal class OCRDecoder + { + private string _configFile; + private string _imagesFolder; + private List _drivers; + private List _mainZones; + private Bitmap FullImage; + public string ConfigFile { get => _configFile; private set => _configFile = value; } + public string ImagesFolder { get => _imagesFolder; private set => _imagesFolder = value; } + public List Drivers { get => _drivers; private set => _drivers = value; } + public List MainZones { get => _mainZones; set => _mainZones = value; } + + //All the image infos will be deleted in not too much time when the merge with the program that recovers the images + const string DEFAULT_IMAGE_NAME = "screen_"; + // You will defenitely have to change this if you want to be able to see debug images + public const string DEBUG_DUMP_FOLDER = @"C:\Users\Moi\Desktop\imgDump\Decode\"; + const int NUMBER_OF_DRIVERS = 20; + + public OCRDecoder(string configFile, string imageFolder) + { + ConfigFile = configFile; + ImagesFolder = imageFolder; + Load(82); + } + /// + /// Method that reads the JSON config file and create all the Zones and Windows + /// + /// The image #id on wich you want to create the zones on + private void Load(int imageNumber) + { + MainZones = new List(); + try + { + FullImage = (Bitmap)Image.FromFile(ImagesFolder + DEFAULT_IMAGE_NAME + imageNumber + ".png"); + } + catch + { + MessageBox.Show("Trouble reaching the image"); + //Maybe a bit to harsh, Ill see what I can do to soft this a bit + Application.Exit(); + } + + Zone MainZone; + try + { + using (var streamReader = new StreamReader(ConfigFile)) + { + var jsonText = streamReader.ReadToEnd(); + var jsonDocument = JsonDocument.Parse(jsonText); + + var driversNames = jsonDocument.RootElement.GetProperty("Drivers"); + Drivers = new List(); + + foreach (var nameElement in driversNames.EnumerateArray()) + { + Drivers.Add(nameElement.GetString()); + } + + var mainProperty = jsonDocument.RootElement.GetProperty("Main"); + Point MainPosition = new Point(mainProperty.GetProperty("x").GetInt32(), mainProperty.GetProperty("y").GetInt32()); + Size MainSize = new Size(mainProperty.GetProperty("width").GetInt32(), mainProperty.GetProperty("height").GetInt32()); + Rectangle MainRectangle = new Rectangle(MainPosition, MainSize); + MainZone = new Zone(FullImage, MainRectangle); + + var zones = mainProperty.GetProperty("Zones"); + var driverZone = zones[0].GetProperty("DriverZone"); + + Point FirstZonePosition = new Point(driverZone.GetProperty("x").GetInt32(), driverZone.GetProperty("y").GetInt32()); + Size FirstZoneSize = new Size(driverZone.GetProperty("width").GetInt32(), driverZone.GetProperty("height").GetInt32()); + + var windows = driverZone.GetProperty("Windows"); + + //var driverPosition = windows.GetProperty("Position"); + var driverPosition = windows[0].GetProperty("Position"); + Size driverPositionArea = new Size(driverPosition.GetProperty("width").GetInt32(), FirstZoneSize.Height); + Point driverPositionPosition = new Point(driverPosition.GetProperty("x").GetInt32(), driverPosition.GetProperty("y").GetInt32()); + + var driverGapToLeader = windows[0].GetProperty("GapToLeader"); + Size driverGapToLeaderArea = new Size(driverGapToLeader.GetProperty("width").GetInt32(), FirstZoneSize.Height); + Point driverGapToLeaderPosition = new Point(driverGapToLeader.GetProperty("x").GetInt32(), driverGapToLeader.GetProperty("y").GetInt32()); + + var driverLapTime = windows[0].GetProperty("LapTime"); + Size driverLapTimeArea = new Size(driverLapTime.GetProperty("width").GetInt32(), FirstZoneSize.Height); + Point driverLapTimePosition = new Point(driverLapTime.GetProperty("x").GetInt32(), driverLapTime.GetProperty("y").GetInt32()); + + + var driverDrs = windows[0].GetProperty("Drs"); + Size driverDrsArea = new Size(driverDrs.GetProperty("width").GetInt32(), FirstZoneSize.Height); + Point driverDrsPosition = new Point(driverDrs.GetProperty("x").GetInt32(), driverDrs.GetProperty("y").GetInt32()); + + var driverTyres = windows[0].GetProperty("Tyres"); + Size driverTyresArea = new Size(driverTyres.GetProperty("width").GetInt32(), FirstZoneSize.Height); + Point driverTyresPosition = new Point(driverTyres.GetProperty("x").GetInt32(), driverTyres.GetProperty("y").GetInt32()); + + var driverName = windows[0].GetProperty("Name"); + Size driverNameArea = new Size(driverName.GetProperty("width").GetInt32(), FirstZoneSize.Height); + Point driverNamePosition = new Point(driverName.GetProperty("x").GetInt32(), driverName.GetProperty("y").GetInt32()); + + var driverSector1 = windows[0].GetProperty("Sector1"); + Size driverSector1Area = new Size(driverSector1.GetProperty("width").GetInt32(), FirstZoneSize.Height); + Point driverSector1Position = new Point(driverSector1.GetProperty("x").GetInt32(), driverSector1.GetProperty("y").GetInt32()); + + var driverSector2 = windows[0].GetProperty("Sector2"); + Size driverSector2Area = new Size(driverSector2.GetProperty("width").GetInt32(), FirstZoneSize.Height); + Point driverSector2Position = new Point(driverSector2.GetProperty("x").GetInt32(), driverSector2.GetProperty("y").GetInt32()); + + var driverSector3 = windows[0].GetProperty("Sector3"); + Size driverSector3Area = new Size(driverSector3.GetProperty("width").GetInt32(), FirstZoneSize.Height); + Point driverSector3Position = new Point(driverSector3.GetProperty("x").GetInt32(), driverSector3.GetProperty("y").GetInt32()); + + float offset = (((float)MainZone.ZoneImage.Height - (float)(Drivers.Count * FirstZoneSize.Height)) / (float)Drivers.Count); + Bitmap MainZoneImage = MainZone.ZoneImage; + List zonesToAdd = new List(); + List zonesImages = new List(); + + for (int i = 0; i < NUMBER_OF_DRIVERS; i++) + { + Point tmpPos = new Point(0, FirstZonePosition.Y + i * FirstZoneSize.Height - Convert.ToInt32(i * offset)); + Zone newDriverZone = new Zone(MainZoneImage, new Rectangle(tmpPos, FirstZoneSize)); + zonesToAdd.Add(newDriverZone); + zonesImages.Add(newDriverZone.ZoneImage); + } + + //Parallel.For(0, NUMBER_OF_DRIVERS, i => + for (int i = 0; i < NUMBER_OF_DRIVERS; i++) + { + Zone newDriverZone = zonesToAdd[(int)i]; + Bitmap zoneImg = zonesImages[(int)i]; + + newDriverZone.AddWindow(new DriverPositionWindow(zoneImg, new Rectangle(driverPositionPosition, driverPositionArea))); + newDriverZone.AddWindow(new DriverGapToLeaderWindow(zoneImg, new Rectangle(driverGapToLeaderPosition, driverGapToLeaderArea))); + newDriverZone.AddWindow(new DriverLapTimeWindow(zoneImg, new Rectangle(driverLapTimePosition, driverLapTimeArea))); + newDriverZone.AddWindow(new DriverDrsWindow(zoneImg, new Rectangle(driverDrsPosition, driverDrsArea))); + newDriverZone.AddWindow(new DriverTyresWindow(zoneImg, new Rectangle(driverTyresPosition, driverTyresArea))); + newDriverZone.AddWindow(new DriverNameWindow(zoneImg, new Rectangle(driverNamePosition, driverNameArea))); + newDriverZone.AddWindow(new DriverSector1Window(zoneImg, new Rectangle(driverSector1Position, driverSector1Area))); + newDriverZone.AddWindow(new DriverSector2Window(zoneImg, new Rectangle(driverSector2Position, driverSector2Area))); + newDriverZone.AddWindow(new DriverSector3Window(zoneImg, new Rectangle(driverSector3Position, driverSector3Area))); + + MainZone.AddZone(newDriverZone); + }//); + //MessageBox.Show("We have a main zone with " + MainZone.Zones.Count() + " Driver zones with " + MainZone.Zones[4].Windows.Count() + " windows each and we have " + Drivers.Count + " drivers"); + MainZones.Add(MainZone); + } + } + catch (IOException ex) + { + MessageBox.Show("Error reading JSON file: " + ex.Message); + } + catch (JsonException ex) + { + MessageBox.Show("Invalid JSON format: " + ex.Message); + } + } + /// + /// Changes the zones and windows to the new image + /// + /// The #id of the new image on wich to do OCR + public void ChangeImage(int imageNumber) + { + Bitmap img = null; + string imagePath = ImagesFolder + DEFAULT_IMAGE_NAME + imageNumber + ".png"; + try + { + img = (Bitmap)Image.FromFile(imagePath); + } + catch + { + MessageBox.Show("Unable to reach the image at " + imagePath); + } + if (img != null) + { + FullImage = img; + foreach (Zone z in MainZones) + { + z.Image = img; + } + } + } + /// + /// Method that calls all the zones and windows to get the content they can find on the image to display them + /// + /// The id of the image we are working with + /// a string representation of all the returns + public async Task Decode(int idImage) + { + string result = ""; + ChangeImage(idImage); + List mainResults = new List(); + + //Decode + for (int mainZoneId = 0; mainZoneId < MainZones.Count; mainZoneId++) + { + switch (mainZoneId) + { + case 0: + //Main Zone + foreach (Zone z in MainZones[mainZoneId].Zones) + { + mainResults.Add(await z.Decode(Drivers)); + } + break; + //Next there could be a Title Zone and TrackInfoZone + } + } + + //Display + foreach (DriverData driver in mainResults) + { + result += driver.ToString(); + result += Environment.NewLine; + } + + return result; + } + /// + /// Method that can be used to convert an amount of miliseconds into a more readable human form + /// + /// The given amount of miliseconds ton convert + /// A human readable string that represents the ms + public static string ConvertMsToTime(int amountOfMs) + { + //Convert.ToInt32 would round upand I dont want that + int minuts = (int)((float)amountOfMs / (1000f * 60f)); + int seconds = (int)((amountOfMs - (minuts * 60f * 1000f)) / 1000); + int ms = amountOfMs - ((minuts * 60 * 1000) + (seconds * 1000)); + + return minuts + ":" + seconds.ToString("00") + ":" + ms.ToString("000"); + } + /// + /// Old method that can draw on an image where the windows and zones are created. mostly used for debugging + /// + /// the #id of the image we are working with + /// the drawed bitmap + public Bitmap Draw(int idImage) + { + Bitmap result; + try + { + result = (Bitmap)Image.FromFile(ImagesFolder + DEFAULT_IMAGE_NAME + idImage + ".png"); + } + catch + { + MessageBox.Show("Image could not be found"); + return null; + } + + Graphics g = Graphics.FromImage(result); + + foreach (Zone z in MainZones) + { + int count = 0; + foreach (Zone zz in z.Zones) + { + g.DrawRectangle(Pens.Red, z.Bounds); + foreach (Window w in zz.Windows) + { + g.DrawRectangle(Pens.Blue, new Rectangle(z.Bounds.X + zz.Bounds.X, z.Bounds.Y + zz.Bounds.Y, zz.Bounds.Width, zz.Bounds.Height)); + } + + count++; + } + } + + return result; + } + } +} diff --git a/Test_Merge/OcrImage.cs b/Test_Merge/OcrImage.cs new file mode 100644 index 0000000..7b02432 --- /dev/null +++ b/Test_Merge/OcrImage.cs @@ -0,0 +1,533 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using System.Drawing; +using System.Drawing.Drawing2D; +using System.Drawing.Imaging; + +namespace Test_Merge +{ + public class OcrImage + { + //this is a hardcoded value based on the colors of the F1TV data channel background you can change it if sometime in the future the color changes + //Any color that has any of its R,G or B channel higher than the treshold will be considered as being usefull information + public static Color F1TV_BACKGROUND_TRESHOLD = Color.FromArgb(0x50, 0x50, 0x50); + Bitmap InputBitmap; + public enum WindowType + { + LapTime, + Text, + Sector, + Gap, + Tyre, + } + + /// + /// Create a new Ocr image to help enhance the given bitmap for OCR + /// + /// The image you want to enhance + public OcrImage(Bitmap inputBitmap) + { + InputBitmap = inputBitmap; + } + /// + /// Enhances the image depending on wich type of window the image comes from + /// + /// The type of the window. Depending on it different enhancing features will be applied + /// The enhanced Bitmap + public Bitmap Enhance(WindowType type = WindowType.Text) + { + Bitmap outputBitmap = (Bitmap)InputBitmap.Clone(); + switch (type) + { + case WindowType.LapTime: + outputBitmap = Tresholding(outputBitmap, 185); + outputBitmap = Resize(outputBitmap, 2); + outputBitmap = Dilatation(outputBitmap, 1); + outputBitmap = Erode(outputBitmap, 1); + break; + case WindowType.Text: + outputBitmap = InvertColors(outputBitmap); + outputBitmap = Tresholding(outputBitmap, 165); + outputBitmap = Resize(outputBitmap, 2); + outputBitmap = Dilatation(outputBitmap, 1); + break; + case WindowType.Tyre: + outputBitmap = RemoveUseless(outputBitmap); + outputBitmap = Resize(outputBitmap, 4); + outputBitmap = Dilatation(outputBitmap, 1); + break; + default: + outputBitmap = Tresholding(outputBitmap, 165); + outputBitmap = Resize(outputBitmap, 4); + outputBitmap = Erode(outputBitmap, 1); + break; + } + return outputBitmap; + } + /// + /// Method that convert a colored RGB bitmap into a GrayScale image + /// + /// The Bitmap you want to convert + /// The bitmap in grayscale + public static Bitmap Grayscale(Bitmap inputBitmap) + { + Rectangle rect = new Rectangle(0, 0, inputBitmap.Width, inputBitmap.Height); + BitmapData bmpData = inputBitmap.LockBits(rect, ImageLockMode.ReadWrite, inputBitmap.PixelFormat); + int bytesPerPixel = Bitmap.GetPixelFormatSize(inputBitmap.PixelFormat) / 8; + + unsafe + { + byte* ptr = (byte*)bmpData.Scan0.ToPointer(); + for (int y = 0; y < inputBitmap.Height; y++) + { + byte* currentLine = ptr + (y * bmpData.Stride); + for (int x = 0; x < inputBitmap.Width; x++) + { + byte* pixel = currentLine + (x * bytesPerPixel); + + byte blue = pixel[0]; + byte green = pixel[1]; + byte red = pixel[2]; + + //Those a specific values to correct the weights so its more pleasing to the human eye + int gray = (int)(red * 0.3 + green * 0.59 + blue * 0.11); + + pixel[0] = pixel[1] = pixel[2] = (byte)gray; + } + } + } + inputBitmap.UnlockBits(bmpData); + + return inputBitmap; + } + /// + /// Method that binaries the input image up to a certain treshold given + /// + /// the bitmap you want to convert to binary colors + /// The floor at wich the color is considered as white or black + /// The binarised bitmap + public static Bitmap Tresholding(Bitmap inputBitmap, int threshold) + { + Rectangle rect = new Rectangle(0, 0, inputBitmap.Width, inputBitmap.Height); + BitmapData bmpData = inputBitmap.LockBits(rect, ImageLockMode.ReadWrite, inputBitmap.PixelFormat); + int bytesPerPixel = Bitmap.GetPixelFormatSize(inputBitmap.PixelFormat) / 8; + + unsafe + { + byte* ptr = (byte*)bmpData.Scan0.ToPointer(); + int bmpHeight = inputBitmap.Height; + int bmpWidth = inputBitmap.Width; + Parallel.For(0, bmpHeight, y => + { + byte* currentLine = ptr + (y * bmpData.Stride); + for (int x = 0; x < bmpWidth; x++) + { + byte* pixel = currentLine + (x * bytesPerPixel); + + byte blue = pixel[0]; + byte green = pixel[1]; + byte red = pixel[2]; + //Those a specific values to correct the weights so its more pleasing to the human eye + int gray = (int)(red * 0.3 + green * 0.59 + blue * 0.11); + int value = gray < threshold ? 0 : 255; + + pixel[0] = pixel[1] = pixel[2] = (byte)value; + } + }); + } + inputBitmap.UnlockBits(bmpData); + + return inputBitmap; + } + /// + /// Method that removes the pixels that are flagged as background + /// + /// The bitmap you want to remove the background from + /// The Bitmap without the background + public static Bitmap RemoveBG(Bitmap inputBitmap) + { + Rectangle rect = new Rectangle(0, 0, inputBitmap.Width, inputBitmap.Height); + BitmapData bmpData = inputBitmap.LockBits(rect, ImageLockMode.ReadWrite, inputBitmap.PixelFormat); + int bytesPerPixel = Bitmap.GetPixelFormatSize(inputBitmap.PixelFormat) / 8; + + unsafe + { + byte* ptr = (byte*)bmpData.Scan0.ToPointer(); + for (int y = 0; y < inputBitmap.Height; y++) + { + byte* currentLine = ptr + (y * bmpData.Stride); + for (int x = 0; x < inputBitmap.Width; x++) + { + byte* pixel = currentLine + (x * bytesPerPixel); + + int B = pixel[0]; + int G = pixel[1]; + int R = pixel[2]; + + if (R <= F1TV_BACKGROUND_TRESHOLD.R && G <= F1TV_BACKGROUND_TRESHOLD.G && B <= F1TV_BACKGROUND_TRESHOLD.B) + pixel[0] = pixel[1] = pixel[2] = 0; + } + } + } + inputBitmap.UnlockBits(bmpData); + + return inputBitmap; + } + /// + /// Method that removes all the useless things from the image and returns hopefully only the numbers + /// + /// The bitmap you want to remove useless things from (Expects a cropped part of the TyreWindow) + /// The bitmap with (hopefully) only the digits + public unsafe static Bitmap RemoveUseless(Bitmap inputBitmap) + { + //Note you can use something else than a cropped tyre window but I would recommend checking the code first to see if it fits your intended use + Rectangle rect = new Rectangle(0, 0, inputBitmap.Width, inputBitmap.Height); + BitmapData bmpData = inputBitmap.LockBits(rect, ImageLockMode.ReadWrite, inputBitmap.PixelFormat); + int bytesPerPixel = Bitmap.GetPixelFormatSize(inputBitmap.PixelFormat) / 8; + + byte* ptr = (byte*)bmpData.Scan0.ToPointer(); + for (int y = 0; y < inputBitmap.Height; y++) + { + byte* currentLine = ptr + (y * bmpData.Stride); + + List pixelsToRemove = new List(); + + bool fromBorder = true; + + for (int x = 0; x < inputBitmap.Width; x++) + { + byte* pixel = currentLine + (x * bytesPerPixel); + + int B = pixel[0]; + int G = pixel[1]; + int R = pixel[2]; + + if (fromBorder && B < F1TV_BACKGROUND_TRESHOLD.B && G < F1TV_BACKGROUND_TRESHOLD.G && R < F1TV_BACKGROUND_TRESHOLD.R) + { + pixelsToRemove.Add(x); + } + else + { + if (fromBorder) + { + fromBorder = false; + pixelsToRemove.Add(x); + } + } + } + fromBorder = true; + for (int x = inputBitmap.Width - 1; x > 0; x--) + { + byte* pixel = currentLine + (x * bytesPerPixel); + + int B = pixel[0]; + int G = pixel[1]; + int R = pixel[2]; + + if (fromBorder && B < F1TV_BACKGROUND_TRESHOLD.B && G < F1TV_BACKGROUND_TRESHOLD.G && R < F1TV_BACKGROUND_TRESHOLD.R) + { + pixelsToRemove.Add(x); + } + else + { + if (fromBorder) + { + fromBorder = false; + pixelsToRemove.Add(x); + } + } + } + + foreach (int pxPos in pixelsToRemove) + { + byte* pixel = currentLine + (pxPos * bytesPerPixel); + + pixel[0] = 0xFF; + pixel[1] = 0xFF; + pixel[2] = 0xFF; + } + } + + //Removing the color parts + for (int y = 0; y < inputBitmap.Height; y++) + { + byte* currentLine = ptr + (y * bmpData.Stride); + for (int x = 0; x < inputBitmap.Width; x++) + { + byte* pixel = currentLine + (x * bytesPerPixel); + + int B = pixel[0]; + int G = pixel[1]; + int R = pixel[2]; + + if (R >= F1TV_BACKGROUND_TRESHOLD.R + 15 || G >= F1TV_BACKGROUND_TRESHOLD.G + 15 || B >= F1TV_BACKGROUND_TRESHOLD.B + 15) + { + pixel[0] = 0xFF; + pixel[1] = 0xFF; + pixel[2] = 0xFF; + } + } + } + + inputBitmap.UnlockBits(bmpData); + return inputBitmap; + } + /// + /// Recovers the average colors from the Image. NOTE : It wont take in account colors that are lower than the background + /// + /// The bitmap you want to get the average color from + /// The average color of the bitmap + public static Color GetAvgColorFromBitmap(Bitmap inputBitmap) + { + Rectangle rect = new Rectangle(0, 0, inputBitmap.Width, inputBitmap.Height); + BitmapData bmpData = inputBitmap.LockBits(rect, ImageLockMode.ReadWrite, inputBitmap.PixelFormat); + int bytesPerPixel = Bitmap.GetPixelFormatSize(inputBitmap.PixelFormat) / 8; + + int totR = 0; + int totG = 0; + int totB = 0; + + int totPixels = 1; + + unsafe + { + byte* ptr = (byte*)bmpData.Scan0.ToPointer(); + int bmpHeight = inputBitmap.Height; + int bmpWidth = inputBitmap.Width; + Parallel.For(0, bmpHeight, y => + { + byte* currentLine = ptr + (y * bmpData.Stride); + for (int x = 0; x < bmpWidth; x++) + { + byte* pixel = currentLine + (x * bytesPerPixel); + + int B = pixel[0]; + int G = pixel[1]; + int R = pixel[2]; + + if (R >= F1TV_BACKGROUND_TRESHOLD.R || G >= F1TV_BACKGROUND_TRESHOLD.G || B >= F1TV_BACKGROUND_TRESHOLD.B) + { + totPixels++; + totB += pixel[0]; + totG += pixel[1]; + totR += pixel[2]; + } + } + }); + } + inputBitmap.UnlockBits(bmpData); + + return Color.FromArgb(255, Convert.ToInt32((float)totR / (float)totPixels), Convert.ToInt32((float)totG / (float)totPixels), Convert.ToInt32((float)totB / (float)totPixels)); + } + /// + /// This method simply inverts all the colors in a Bitmap + /// + /// the bitmap you want to invert the colors from + /// The bitmap with inverted colors + public static Bitmap InvertColors(Bitmap inputBitmap) + { + Rectangle rect = new Rectangle(0, 0, inputBitmap.Width, inputBitmap.Height); + BitmapData bmpData = inputBitmap.LockBits(rect, ImageLockMode.ReadWrite, inputBitmap.PixelFormat); + int bytesPerPixel = Bitmap.GetPixelFormatSize(inputBitmap.PixelFormat) / 8; + + unsafe + { + byte* ptr = (byte*)bmpData.Scan0.ToPointer(); + for (int y = 0; y < inputBitmap.Height; y++) + { + byte* currentLine = ptr + (y * bmpData.Stride); + for (int x = 0; x < inputBitmap.Width; x++) + { + byte* pixel = currentLine + (x * bytesPerPixel); + + pixel[0] = (byte)(255 - pixel[0]); + pixel[1] = (byte)(255 - pixel[1]); + pixel[2] = (byte)(255 - pixel[2]); + } + } + } + inputBitmap.UnlockBits(bmpData); + + return inputBitmap; + } + /// + /// Methods that applies Bicubic interpolation to increase the size and resolution of an image + /// + /// The bitmap you want to resize + /// The factor of resizing you want to use. I recommend using even numbers + /// The bitmap witht the new size + public static Bitmap Resize(Bitmap inputBitmap, int resizeFactor) + { + var resultBitmap = new Bitmap(inputBitmap.Width * resizeFactor, inputBitmap.Height * resizeFactor); + + using (var graphics = Graphics.FromImage(resultBitmap)) + { + graphics.InterpolationMode = InterpolationMode.HighQualityBicubic; + graphics.DrawImage(inputBitmap, new Rectangle(0, 0, resultBitmap.Width, resultBitmap.Height)); + } + + return resultBitmap; + } + /// + /// method that Highlights the countours of a Bitmap + /// + /// The bitmap you want to highlight the countours of + /// The bitmap with countours highlighted + public static Bitmap HighlightContours(Bitmap inputBitmap) + { + Bitmap outputBitmap = new Bitmap(inputBitmap.Width, inputBitmap.Height); + + Bitmap grayscale = Grayscale(inputBitmap); + Bitmap thresholded = Tresholding(grayscale, 128); + Bitmap dilated = Dilatation(thresholded, 3); + Bitmap eroded = Erode(dilated, 3); + + for (int y = 0; y < inputBitmap.Height; y++) + { + for (int x = 0; x < inputBitmap.Width; x++) + { + Color pixel = inputBitmap.GetPixel(x, y); + Color dilatedPixel = dilated.GetPixel(x, y); + Color erodedPixel = eroded.GetPixel(x, y); + + int gray = (int)(pixel.R * 0.3 + pixel.G * 0.59 + pixel.B * 0.11); + int threshold = dilatedPixel.R; + + if (gray > threshold) + { + outputBitmap.SetPixel(x, y, Color.FromArgb(255, 255, 255)); + } + else if (gray <= threshold && erodedPixel.R == 0) + { + outputBitmap.SetPixel(x, y, Color.FromArgb(255, 0, 0)); + } + else + { + outputBitmap.SetPixel(x, y, Color.FromArgb(0, 0, 0)); + } + } + } + + return outputBitmap; + } + /// + /// Method that that erodes the morphology of a bitmap + /// + /// The bitmap you want to erode + /// The amount of Erosion you want (be carefull its expensive on ressources) + /// The Bitmap with the eroded contents + public static Bitmap Erode(Bitmap inputBitmap, int kernelSize) + { + Bitmap outputBitmap = new Bitmap(inputBitmap.Width, inputBitmap.Height); + + int[,] kernel = new int[kernelSize, kernelSize]; + + for (int i = 0; i < kernelSize; i++) + { + for (int j = 0; j < kernelSize; j++) + { + kernel[i, j] = 1; + } + } + + for (int y = kernelSize / 2; y < inputBitmap.Height - kernelSize / 2; y++) + { + for (int x = kernelSize / 2; x < inputBitmap.Width - kernelSize / 2; x++) + { + bool flag = true; + + for (int i = -kernelSize / 2; i <= kernelSize / 2; i++) + { + for (int j = -kernelSize / 2; j <= kernelSize / 2; j++) + { + Color pixel = inputBitmap.GetPixel(x + i, y + j); + int gray = (int)(pixel.R * 0.3 + pixel.G * 0.59 + pixel.B * 0.11); + + if (gray >= 128 && kernel[i + kernelSize / 2, j + kernelSize / 2] == 1) + { + flag = false; + break; + } + } + + if (!flag) + { + break; + } + } + + if (flag) + { + outputBitmap.SetPixel(x, y, Color.FromArgb(255, 255, 255)); + } + else + { + outputBitmap.SetPixel(x, y, Color.FromArgb(0, 0, 0)); + } + } + } + + return outputBitmap; + } + /// + /// Method that that use dilatation of the morphology of a bitmap + /// + /// The bitmap you want to use dilatation on + /// The amount of dilatation you want (be carefull its expensive on ressources) + /// The Bitmap after Dilatation + public static Bitmap Dilatation(Bitmap inputBitmap, int kernelSize) + { + Bitmap outputBitmap = new Bitmap(inputBitmap.Width, inputBitmap.Height); + + int[,] kernel = new int[kernelSize, kernelSize]; + + for (int i = 0; i < kernelSize; i++) + { + for (int j = 0; j < kernelSize; j++) + { + kernel[i, j] = 1; + } + } + + for (int y = kernelSize / 2; y < inputBitmap.Height - kernelSize / 2; y++) + { + for (int x = kernelSize / 2; x < inputBitmap.Width - kernelSize / 2; x++) + { + bool flag = false; + + for (int i = -kernelSize / 2; i <= kernelSize / 2; i++) + { + for (int j = -kernelSize / 2; j <= kernelSize / 2; j++) + { + Color pixel = inputBitmap.GetPixel(x + i, y + j); + int gray = (int)(pixel.R * 0.3 + pixel.G * 0.59 + pixel.B * 0.11); + + if (gray < 128 && kernel[i + kernelSize / 2, j + kernelSize / 2] == 1) + { + flag = true; + break; + } + } + + if (flag) + { + break; + } + } + + if (flag) + { + outputBitmap.SetPixel(x, y, Color.FromArgb(0, 0, 0)); + } + else + { + outputBitmap.SetPixel(x, y, Color.FromArgb(255, 255, 255)); + } + } + } + + return outputBitmap; + } + } +} diff --git a/Test_Merge/Settings.cs b/Test_Merge/Settings.cs index 78aa4e3..d7f07e8 100644 --- a/Test_Merge/Settings.cs +++ b/Test_Merge/Settings.cs @@ -261,6 +261,7 @@ namespace Test_Merge private async void btnRefresh_Click(object sender, EventArgs e) { + btnRefresh.Enabled = false; if (Emulator == null) { Emulator = new F1TVEmulator(tbxGpUrl.Text); @@ -268,9 +269,30 @@ namespace Test_Merge if (!Emulator.Ready) { - if (await Emulator.Start() != 0) + Task start = Task.Run(() => Emulator.Start()); + int errorCode = await start; + if (errorCode != 0) { - MessageBox.Show("Could not start the emulator"); + string message; + switch (errorCode) + { + case 101: + message = "Error " + errorCode + " Could not start the driver. It could be because an other instance is runnin make sure you closed them all before trying again"; + break; + case 102: + message = "Error " + errorCode + " Could not navigate on the F1TV site. Make sure the correct URL has been given and that you logged from chrome. It can take a few minutes to update"; + break; + case 103: + message = "Error "+errorCode+" The url is not a valid url"; + break; + case 104: + message = "Error "+errorCode+" The url is not a valid url"; + break; + default: + message = "Could not start the emulator Error " + errorCode; + break; + } + MessageBox.Show(message); } else { @@ -281,6 +303,7 @@ namespace Test_Merge { pbxMain.Image = Emulator.Screenshot(); } + btnRefresh.Enabled = true; } } } diff --git a/Test_Merge/Test_Merge.csproj b/Test_Merge/Test_Merge.csproj index e6dc786..06ba7e8 100644 --- a/Test_Merge/Test_Merge.csproj +++ b/Test_Merge/Test_Merge.csproj @@ -24,6 +24,7 @@ DEBUG;TRACE prompt 4 + true AnyCPU @@ -35,8 +36,36 @@ 4 + + ..\packages\Microsoft.Bcl.AsyncInterfaces.7.0.0\lib\net462\Microsoft.Bcl.AsyncInterfaces.dll + + + ..\packages\System.Buffers.4.5.1\lib\net461\System.Buffers.dll + + + ..\packages\System.Memory.4.5.5\lib\net461\System.Memory.dll + + + + ..\packages\System.Numerics.Vectors.4.5.0\lib\net46\System.Numerics.Vectors.dll + + + ..\packages\System.Runtime.CompilerServices.Unsafe.6.0.0\lib\net461\System.Runtime.CompilerServices.Unsafe.dll + + + ..\packages\System.Text.Encodings.Web.7.0.0\lib\net462\System.Text.Encodings.Web.dll + + + ..\packages\System.Text.Json.7.0.2\lib\net462\System.Text.Json.dll + + + ..\packages\System.Threading.Tasks.Extensions.4.5.4\lib\net461\System.Threading.Tasks.Extensions.dll + + + ..\packages\System.ValueTuple.4.5.0\lib\net47\System.ValueTuple.dll + @@ -46,6 +75,9 @@ + + ..\packages\Tesseract.5.2.0\lib\net47\Tesseract.dll + ..\packages\Selenium.WebDriver.4.8.2\lib\net47\WebDriver.dll @@ -54,6 +86,7 @@ + Form @@ -61,6 +94,8 @@ Form1.cs + + @@ -69,6 +104,8 @@ Settings.cs + + Form1.cs @@ -106,6 +143,8 @@ + + \ No newline at end of file diff --git a/Test_Merge/Window.cs b/Test_Merge/Window.cs new file mode 100644 index 0000000..44cc5cc --- /dev/null +++ b/Test_Merge/Window.cs @@ -0,0 +1,297 @@ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Drawing; +using System.IO; +using Tesseract; +using System.Text.RegularExpressions; +using System.Drawing.Drawing2D; + +namespace Test_Merge +{ + public class Window + { + private Rectangle _bounds; + private Bitmap _image; + private string _name; + protected TesseractEngine Engine; + public Rectangle Bounds { get => _bounds; private set => _bounds = value; } + public Bitmap Image { get => _image; set => _image = value; } + public string Name { get => _name; protected set => _name = value; } + //This will have to be changed if you want to make it run on your machine + public static DirectoryInfo TESS_DATA_FOLDER = new DirectoryInfo(@"C:\Users\Moi\Pictures\SeleniumScreens\TessData"); + + public Bitmap WindowImage + { + get + { + //This little trickery lets you have the image that the window sees + Bitmap sample = new Bitmap(Bounds.Width, Bounds.Height); + Graphics g = Graphics.FromImage(sample); + g.DrawImage(Image, new Rectangle(0, 0, sample.Width, sample.Height), Bounds, GraphicsUnit.Pixel); + return sample; + } + } + public Window(Bitmap image, Rectangle bounds) + { + Image = image; + Bounds = bounds; + Engine = new TesseractEngine(TESS_DATA_FOLDER.FullName, "eng", EngineMode.Default); + Engine.DefaultPageSegMode = PageSegMode.SingleLine; + } + /// + /// Method that will have to be used by the childrens to let the model make them decode the images they have + /// + /// Returns an object because we dont know what kind of return it will be + public virtual async Task DecodePng() + { + return "NaN"; + } + /// + /// Method that will have to be used by the childrens to let the model make them decode the images they have + /// + /// This is a list of the different possible drivers in the race. It should not be too big but NEVER be too short + /// Returns an object because we dont know what kind of return it will be + public virtual async Task DecodePng(List driverList) + { + return "NaN"; + } + /// + /// This converts an image into a byte[]. It can be usefull when doing unsafe stuff. Use at your own risks + /// + /// The image you want to convert + /// A byte array containing the image informations + public static byte[] ImageToByte(Image inputImage) + { + using (var stream = new MemoryStream()) + { + inputImage.Save(stream, System.Drawing.Imaging.ImageFormat.Png); + return stream.ToArray(); + } + } + /// + /// This method is used to recover a time from a PNG using Tesseract OCR + /// + /// The image where the text is + /// The type of window it is + /// The Tesseract Engine + /// The time in milliseconds + public static async Task GetTimeFromPng(Bitmap windowImage, OcrImage.WindowType windowType, TesseractEngine Engine) + { + //Kind of a big method but it has a lot of error handling and has to work with three special cases + string rawResult = ""; + int result = 0; + + switch (windowType) + { + case OcrImage.WindowType.Sector: + //The usual sector is in this form : 33.456 + Engine.SetVariable("tessedit_char_whitelist", "0123456789."); + break; + case OcrImage.WindowType.LapTime: + //The usual Lap time is in this form : 1:45:345 + Engine.SetVariable("tessedit_char_whitelist", "0123456789.:"); + break; + case OcrImage.WindowType.Gap: + //The usual Gap is in this form : + 34.567 + Engine.SetVariable("tessedit_char_whitelist", "0123456789.+"); + break; + default: + Engine.SetVariable("tessedit_char_whitelist", ""); + break; + } + + + Bitmap enhancedImage = new OcrImage(windowImage).Enhance(windowType); + + var tessImage = Pix.LoadFromMemory(ImageToByte(enhancedImage)); + + Page page = Engine.Process(tessImage); + Graphics g = Graphics.FromImage(enhancedImage); + // Get the iterator for the page layout + using (var iter = page.GetIterator()) + { + // Loop over the elements of the page layout + iter.Begin(); + do + { + // Get the text for the current element + try + { + rawResult += iter.GetText(PageIteratorLevel.Word); + } + catch + { + //nothing we just dont add it if its not a number + } + } while (iter.Next(PageIteratorLevel.Word)); + } + + List rawNumbers; + + //In the gaps we can find '+' but we dont care about it its redondant a driver will never be - something + if (windowType == OcrImage.WindowType.Gap) + rawResult = Regex.Replace(rawResult, "[^0-9.:]", ""); + + //Splits into minuts seconds miliseconds + rawNumbers = rawResult.Split('.', ':').ToList(); + //removes any empty cells (tho this usually sign of a really bad OCR implementation tbh will have to be fixed higher in the chian) + rawNumbers.RemoveAll(x => ((string)x) == ""); + + if (rawNumbers.Count == 3) + { + //mm:ss:ms + result = (Convert.ToInt32(rawNumbers[0]) * 1000 * 60) + (Convert.ToInt32(rawNumbers[1]) * 1000) + Convert.ToInt32(rawNumbers[2]); + } + else + { + if (rawNumbers.Count == 2) + { + //ss:ms + result = (Convert.ToInt32(rawNumbers[0]) * 1000) + Convert.ToInt32(rawNumbers[1]); + + if (result > 999999) + { + //We know that we have way too much seconds to make a minut + //Its usually because the ":" have been interpreted as a number + int minuts = (int)(rawNumbers[0][0] - '0'); + // rawNumbers[0][1] should contain the : that has been mistaken + int seconds = Convert.ToInt32(rawNumbers[0][2].ToString() + rawNumbers[0][3].ToString()); + int ms = Convert.ToInt32(rawNumbers[1]); + result = (Convert.ToInt32(minuts) * 1000 * 60) + (Convert.ToInt32(seconds) * 1000) + Convert.ToInt32(ms); + } + } + else + { + if (rawNumbers.Count == 1) + { + try + { + result = Convert.ToInt32(rawNumbers[0]); + } + catch + { + //It can be because the input is empty or because its the LEADER bracket + result = 0; + } + } + else + { + //Auuuugh + result = 0; + } + } + } + page.Dispose(); + return result; + } + /// + /// Method that recovers strings from an image using Tesseract OCR + /// + /// The image of the window that contains text + /// The Tesseract engine + /// The list of allowed chars + /// The type of window the text is on. Depending on the context the OCR will behave differently + /// the string it found + public static async Task GetStringFromPng(Bitmap WindowImage, TesseractEngine Engine, string allowedChars = "", OcrImage.WindowType windowType = OcrImage.WindowType.Text) + { + string result = ""; + + Engine.SetVariable("tessedit_char_whitelist", allowedChars); + + Bitmap rawData = WindowImage; + Bitmap enhancedImage = new OcrImage(rawData).Enhance(windowType); + + Page page = Engine.Process(enhancedImage); + using (var iter = page.GetIterator()) + { + iter.Begin(); + do + { + result += iter.GetText(PageIteratorLevel.Word); + } while (iter.Next(PageIteratorLevel.Word)); + } + page.Dispose(); + return result; + } + /// + /// Get a smaller image from a bigger one + /// + /// The big bitmap you want to get a part of + /// The dimensions of the new bitmap + /// The little bitmap + protected Bitmap GetSmallBitmapFromBigOne(Bitmap inputBitmap, Rectangle newBitmapDimensions) + { + Bitmap sample = new Bitmap(newBitmapDimensions.Width, newBitmapDimensions.Height); + Graphics g = Graphics.FromImage(sample); + g.DrawImage(inputBitmap, new Rectangle(0, 0, sample.Width, sample.Height), newBitmapDimensions, GraphicsUnit.Pixel); + return sample; + } + /// + /// Returns the closest string from a list of options + /// + /// an array of all the possibilities + /// the string you want to compare + /// The closest option + protected static string FindClosestMatch(List options, string testString) + { + var closestMatch = ""; + var closestDistance = int.MaxValue; + + foreach (var item in options) + { + var distance = LevenshteinDistance(item, testString); + if (distance < closestDistance) + { + closestMatch = item; + closestDistance = distance; + } + } + return closestMatch; + } + //This method has been generated with the help of ChatGPT + /// + /// Method that computes a score of distance between two strings + /// + /// The first string (order irrelevant) + /// The second string (order irrelevant) + /// The levenshtein distance + protected static int LevenshteinDistance(string string1, string string2) + { + if (string.IsNullOrEmpty(string1)) + { + return string.IsNullOrEmpty(string2) ? 0 : string2.Length; + } + + if (string.IsNullOrEmpty(string2)) + { + return string.IsNullOrEmpty(string1) ? 0 : string1.Length; + } + + var d = new int[string1.Length + 1, string2.Length + 1]; + for (var i = 0; i <= string1.Length; i++) + { + d[i, 0] = i; + } + + for (var j = 0; j <= string2.Length; j++) + { + d[0, j] = j; + } + + for (var i = 1; i <= string1.Length; i++) + { + for (var j = 1; j <= string2.Length; j++) + { + var cost = (string1[i - 1] == string2[j - 1]) ? 0 : 1; + d[i, j] = Math.Min(Math.Min(d[i - 1, j] + 1, d[i, j - 1] + 1), d[i - 1, j - 1] + cost); + } + } + + return d[string1.Length, string2.Length]; + } + } +} diff --git a/Test_Merge/Zone.cs b/Test_Merge/Zone.cs new file mode 100644 index 0000000..30dcc19 --- /dev/null +++ b/Test_Merge/Zone.cs @@ -0,0 +1,108 @@ +using System; +using System.Collections.Generic; +using System.Drawing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Test_Merge +{ + public class Zone + { + private Rectangle _bounds; + private List _zones; + private List _windows; + private Bitmap _image; + + public Bitmap ZoneImage + { + get + { + //This little trickery lets you have the image that the zone sees + Bitmap sample = new Bitmap(Bounds.Width, Bounds.Height); + Graphics g = Graphics.FromImage(sample); + g.DrawImage(Image, new Rectangle(0, 0, sample.Width, sample.Height), Bounds, GraphicsUnit.Pixel); + return sample; + } + } + public Bitmap Image + { + get { return _image; } + set + { + //It automatically sets the image for the contained windows and zones + _image = Image; + foreach (Window w in Windows) + { + w.Image = ZoneImage; + } + foreach (Zone z in Zones) + { + z.Image = Image; + } + } + } + + public Rectangle Bounds { get => _bounds; protected set => _bounds = value; } + public List Zones { get => _zones; protected set => _zones = value; } + public List Windows { get => _windows; protected set => _windows = value; } + + public Zone(Bitmap image, Rectangle bounds) + { + Windows = new List(); + Zones = new List(); + + //You cant set the image in the CTOR because the processing is impossible at first initiation + _image = image; + Bounds = bounds; + } + /// + /// Adds a zone to the list of zones + /// + /// The zone you want to add + public virtual void AddZone(Zone zone) + { + Zones.Add(zone); + } + /// + /// Add a window to the list of windows + /// + /// the window you want to add + public virtual void AddWindow(Window window) + { + Windows.Add(window); + } + /// + /// Calls all the windows to do OCR and to give back the results so we can send them to the model + /// + /// A list of all the driver in the race to help with text recognition + /// A driver data object that contains all the infos about a driver + public virtual async Task Decode(List driverList) + { + DriverData result = new DriverData(); + Parallel.ForEach(Windows, async w => + { + // A switch would be prettier but I dont think its supported in this C# version + if (w is DriverNameWindow) + result.Name = (string)await (w as DriverNameWindow).DecodePng(driverList); + if (w is DriverDrsWindow) + result.DRS = (bool)await (w as DriverDrsWindow).DecodePng(); + if (w is DriverGapToLeaderWindow) + result.GapToLeader = (int)await (w as DriverGapToLeaderWindow).DecodePng(); + if (w is DriverLapTimeWindow) + result.LapTime = (int)await (w as DriverLapTimeWindow).DecodePng(); + if (w is DriverPositionWindow) + result.Position = (int)await (w as DriverPositionWindow).DecodePng(); + if (w is DriverSector1Window) + result.Sector1 = (int)await (w as DriverSector1Window).DecodePng(); + if (w is DriverSector2Window) + result.Sector2 = (int)await (w as DriverSector2Window).DecodePng(); + if (w is DriverSector3Window) + result.Sector3 = (int)await (w as DriverSector3Window).DecodePng(); + if (w is DriverTyresWindow) + result.CurrentTyre = (Tyre)await (w as DriverTyresWindow).DecodePng(); + }); + return result; + } + } +} diff --git a/Test_Merge/packages.config b/Test_Merge/packages.config index df1388b..442e636 100644 --- a/Test_Merge/packages.config +++ b/Test_Merge/packages.config @@ -1,6 +1,16 @@  + + + + + + + + + + \ No newline at end of file