From 48e5e92183de91bad1047ec77d56fa1b4eab5f7e Mon Sep 17 00:00:00 2001 From: Dylan Lytle Date: Thu, 22 Feb 2024 22:46:00 -0700 Subject: [PATCH] Pull Request 81 Dynamic Map Selection Maps are dynamically selected by reading the map name text. OCR function simplified and improved for map name detection. Map names are no longer needed to be mapped to page and index in static.py. They only need to be added to setup.json. Spaces and underscores are ignored. Process is as follows: - at each map selection page: - screenshot each of the six map name sections of the images - process the text from the image - use the fuzz library to get a score compared to the map name setting - go back to the page and index with the highest fuzz score and select it - if below a setting threshold throw an exception Debug available: - as with the round text detection, screenshots are saved off - each map index is compared against a list of all map names to verify it is detecting correctly --- src/Bot.py | 101 +++++++++++++++++++++++++++++++++++++++++++--- src/ocr.py | 63 +++-------------------------- src/static.py | 71 +++----------------------------- src/statictest.py | 82 +++++++++++++++++++++++++++++++++++++ 4 files changed, 189 insertions(+), 128 deletions(-) create mode 100644 src/statictest.py diff --git a/src/Bot.py b/src/Bot.py index 2f61702..dd20652 100644 --- a/src/Bot.py +++ b/src/Bot.py @@ -14,6 +14,8 @@ import simulatedinput import monitor import gameplan +import statictest +from fuzzywuzzy import fuzz from logger import logger as log # Definently fix this @@ -481,10 +483,36 @@ def restart_level(self, won=True): self.wait_for_loading() # wait for loading screen - def select_map(self): - map_page = static.maps[self.settings["MAP"]][0] - map_index = static.maps[self.settings["MAP"]][1] + def getMapNameAtIndex(self, index): + # Get the top left and bottom right points + top_left_scaled = monitor.scaling(static.map_selection["MAPS_TOP_LEFT"]) + bottom_right_scaled = monitor.scaling(static.map_selection["MAPS_BOTTOM_RIGHT"]) + # need tuple for monitor.scaling call but just care about scaling the first value + text_height_scaled = monitor.scaling([static.map_selection["MAPS_TEXT_HEIGHT"], 0])[0] + # Setting up screen capture area using top left and bottom right points. Assume evenly spaced 3 x 2 grid of map options + screenshot_dimensions = { + 'top': int(top_left_scaled[1] + (((bottom_right_scaled[1] - top_left_scaled[1]) / 2) * int(index / 3))), + 'left': int(top_left_scaled[0] + (((bottom_right_scaled[0] - top_left_scaled[0]) / 3) * (index % 3))), + 'width': int((bottom_right_scaled[0] - top_left_scaled[0]) / 3), + 'height': int(text_height_scaled), + } + + # Take Screenshot and get text + with mss.mss() as screenshotter: + screenshot = screenshotter.grab(screenshot_dimensions) + found_text, _ocrImage = ocr.getTextFromImage(screenshot) + + if self.DEBUG: + from cv2 import imwrite, IMWRITE_PNG_COMPRESSION + def get_valid_filename(s): + s = str(s).strip().replace(' ', '_') + return re.sub(r'(?u)[^-\w.]', '', s) + imwrite(f"./DEBUG/OCR_DONE_FOUND_{get_valid_filename(found_text)}_{str(time.time())}.png", _ocrImage, [IMWRITE_PNG_COMPRESSION, 0]) + + return found_text + + def select_map(self): time.sleep(1) simulatedinput.click("HOME_MENU_START") @@ -492,10 +520,71 @@ def select_map(self): simulatedinput.click("BEGINNER_SELECTION") # goto first page - # click to the right page - simulatedinput.click("RIGHT_ARROW_SELECTION", amount=(map_page - 1), timeout=0.1) + # save off the first map name to check for wrap around + initial_first_map_name = self.getMapNameAtIndex(0) + current_first_map_name = None # initialize to None so we don't hit break logic on the first loop + map_index_counter = 0 + # tuple with map index and fuzz score + highest_fuzz_score_found = (0, 0) + + # used for debug + map_names_extracted_correctly = 0 + # tuple with map index and fuzz score + highest_fuzz_score_found_in_expected = (0, 0) + + # format map selection setting by removing all spaces and underscores + formatted_map_selection_setting = self.settings["MAP"].replace("_", "").replace(" ", "").upper() + log.debug(f"formatted_map_selection_setting is \"{formatted_map_selection_setting}\"") + + # interate through maps searching for the text. Exit if we get back to the first page or find the map setting + while True: + # if we get back to the first page break out of the loop + if current_first_map_name == initial_first_map_name: + break + # check each map in 3x2 grid + for i in range(6): + map_name = self.getMapNameAtIndex(i) + formatted_map_name = map_name.upper() + + if self.DEBUG: + # only do debug if we have an expected maps at the index + if map_index_counter < len(statictest.expected_maps): + log.debug(f"Found map name \"{map_name}\" at index {i}") + + # loop through all the expected maps checking making sure the highest fuzz score is at the expected index + for expected_map_index in range(len(statictest.expected_maps)): + formatted_expected_map_name = statictest.expected_maps[expected_map_index].replace("_", "").upper() + fuzz_score = fuzz.partial_ratio(formatted_map_name, formatted_expected_map_name) + # keep track of the index of the highest fuzz score found in the expected maps to compare + if fuzz_score > highest_fuzz_score_found_in_expected[1]: + highest_fuzz_score_found_in_expected = (expected_map_index, fuzz_score) + + if highest_fuzz_score_found_in_expected[0] == map_index_counter: + log.debug(f"Correctly identified map at index {map_index_counter}") + map_names_extracted_correctly += 1 + else: + log.debug(f"Highest fuzz score was not on the expected map") + + fuzz_score = fuzz.partial_ratio(formatted_map_name, formatted_map_selection_setting) + if fuzz_score > highest_fuzz_score_found[1]: + highest_fuzz_score_found = (map_index_counter, fuzz_score) + + map_index_counter += 1 + + # click the arrow once + simulatedinput.click("RIGHT_ARROW_SELECTION", amount=1, timeout=0.1) + current_first_map_name = self.getMapNameAtIndex(0) + + if self.DEBUG: + log.debug(f"Found {map_names_extracted_correctly}/{map_index_counter} map names correctly") + + # If we didn't find the map name on any of the pages that match above the threshold then log an error and return + if highest_fuzz_score_found[1] < static.map_selection["MAP_NAME_FUZZ_THRESHOLD"]: + raise Exception(f"Unable to find specified map above threshold {highest_fuzz_score_found[1]} < {static.map_selection['MAP_NAME_FUZZ_THRESHOLD']}") - simulatedinput.click("MAP_INDEX_" + str(map_index)) # Click correct map + # click the arrow once + simulatedinput.click("RIGHT_ARROW_SELECTION", amount=int(highest_fuzz_score_found[0] / 6), timeout=0.1) + simulatedinput.click("MAP_INDEX_" + str(int(highest_fuzz_score_found[0] % 6) + 1)) # Click correct map if self.SANDBOX: simulatedinput.click("EASY_MODE") # Select Difficulty diff --git a/src/ocr.py b/src/ocr.py index e25d4b4..24e09c4 100644 --- a/src/ocr.py +++ b/src/ocr.py @@ -13,62 +13,11 @@ def formatImageOCR(originalScreenshot): screenshot = np.array(originalScreenshot, dtype=np.uint8) - # Get local maximum: - kernelSize = 5 - maxKernel = cv2.getStructuringElement(cv2.MORPH_RECT, (kernelSize, kernelSize)) - localMax = cv2.morphologyEx(screenshot, cv2.MORPH_CLOSE, maxKernel, None, None, 1, cv2.BORDER_REFLECT101) - # Perform gain division - # print(screenshot, localMax) - gainDivision = np.where(localMax == 0, 0, (screenshot / localMax)) - # Clip the values to [0,255] - gainDivision = np.clip((255 * gainDivision), 0, 255) - # Convert the mat type from float to uint8: - gainDivision = gainDivision.astype("uint8") - # Convert RGB to grayscale: - grayscaleImage = cv2.cvtColor(gainDivision, cv2.COLOR_BGR2GRAY) - # Resize image to improve the quality - grayscaleImage = cv2.resize(grayscaleImage,(0,0), fx=3.0, fy=3.0) - # Get binary image via Otsu: - _, final_image = cv2.threshold(grayscaleImage, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) - # cv2.imwrite(f"./DEBUG/OCR_FORMAT_BINARY_IMAGE_{str(time.time())}.png", final_image, [cv2.IMWRITE_PNG_COMPRESSION, 0]) - - # Set kernel (structuring element) size: - kernelSize = 3 - # Set morph operation iterations: - opIterations = 1 - # Get the structuring element: - morphKernel = cv2.getStructuringElement(cv2.MORPH_RECT, (kernelSize, kernelSize)) - # Perform closing: - final_image = cv2.morphologyEx( final_image, cv2.MORPH_CLOSE, morphKernel, None, None, opIterations, cv2.BORDER_REFLECT101 ) - # cv2.imwrite(f"./DEBUG/OCR_FORMAT_BEFORE_FLOOD_{str(time.time())}.png", final_image, [cv2.IMWRITE_PNG_COMPRESSION, 0]) - - # Flood fill (white + black): - cv2.floodFill(final_image, mask=None, seedPoint=(int(0), int(0)), newVal=(255)) - # Invert image so target blobs are colored in white: - final_image = 255 - final_image - # Find the blobs on the binary image: - contours, hierarchy = cv2.findContours(final_image, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - # Process the contours: - for i, c in enumerate(contours): - # Get contour hierarchy: - currentHierarchy = hierarchy[0][i][3] - # Look only for children contours (the holes): - if currentHierarchy != -1: - # Get the contour bounding rectangle: - boundRect = cv2.boundingRect(c) - # Get the dimensions of the bounding rect: - rectX = boundRect[0] - rectY = boundRect[1] - rectWidth = boundRect[2] - rectHeight = boundRect[3] - # Get the center of the contour the will act as - # seed point to the Flood-Filling: - fx = rectX + 0.5 * rectWidth - fy = rectY + 0.5 * rectHeight - # Fill the hole: - cv2.floodFill(final_image, mask=None, seedPoint=(int(fx), int(fy)), newVal=(0)) - # cv2.imwrite(f"./DEBUG/OCR_FLOOD_{i}_{str(time.time())}.png", final_image, [cv2.IMWRITE_PNG_COMPRESSION, 0]) - + grayscaleImage = cv2.cvtColor(screenshot, cv2.COLOR_BGR2GRAY) + binary = cv2.threshold(grayscaleImage, 245, 255, cv2.THRESH_BINARY)[1] + binary = cv2.bitwise_not(binary) + kernel = np.ones((2, 2), np.uint8) + final_image = cv2.dilate(binary, kernel, iterations=1) return final_image @@ -86,5 +35,5 @@ def getTextFromImage(image): # NOTE: This part seems to be buggy # Get current round from screenshot with tesseract - return pytesseract.image_to_string(imageCandidate, config='--psm 7').replace("\n", ""), imageCandidate + return pytesseract.image_to_string(imageCandidate, config='--psm 10 --oem 3 -c tessedit_char_whitelist=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789/').replace("\n", ""), imageCandidate diff --git a/src/static.py b/src/static.py index 8d3fbe0..bf2ac76 100644 --- a/src/static.py +++ b/src/static.py @@ -136,71 +136,12 @@ class Map (Difficulty, Gamemode): "HERO" : "u" } -# "NAME" : [PAGE, INDEX] -maps = { - "MONKEY_MEADOW" : [1, 1], - "TREE_STUMP" : [1, 2], - "TOWN_CENTER" : [1, 3], - "SCRAPYARD" : [1, 4], - "THE_CABIN" : [1, 5], - "RESORT" : [1, 6], - "SKATES" : [2, 1], - "LOTUS_ISLAND" : [2, 2], - "CANDY_FALLS" : [2, 3], - "WINTER_PARK" : [2, 4], - "CARVED" : [2, 5], - "PARK_PATH" : [2, 6], - "ALPINE_RUN" : [3, 1], - "FROZEN_OVER" : [3, 2], - "IN_THE_LOOP" : [3, 3], - "CUBISM" : [3, 4], - "FOUR_CIRCLES" : [3, 5], - "HEDGE" : [3, 6], - "END_OF_THE_ROAD" : [4, 1], - "LOGS" : [4, 2], - "COVERED_GARDEN" : [5, 1], - "QUARRY" : [5, 2], - "QUIET_STREET" : [5, 3], - "BLOONARIUS_PRIME" : [5, 4], - "BALANCE" : [5, 5], - "ENCRYPTED" : [5, 6], - "BAZAAR" : [6, 1], - "ADORAS_TEMPLE" : [6, 2], - "SPRING_SPRING" : [6, 3], - "KARTSNDARTS" : [6, 4], - "MOON_LANDING" : [6, 5], - "HAUNTED" : [6, 6], - "DOWNSTREAM" : [7, 1], - "FIRING_RANGE" : [7, 2], - "CRACKED" : [7, 3], - "STREAMBED" : [7, 4], - "CHUTES" : [7, 5], - "RAKE" : [7, 6], - "SPICE_ISLANDS" : [8, 1], - "MIDNIGHT_MANSION" : [9, 1], - "SUNKEN_COLUMNS" : [9, 2], - "XFACTOR" : [9, 3], - "MESA" : [9, 4], - "GEARED" : [9, 5], - "SPILLWAY" : [9, 6], - "CARGO" : [10, 1], - "PATS_POND" : [10, 2], - "PENINSULA" : [10, 3], - "HIGH_FINANCE" : [10, 4], - "ANOTHER_BRICK" : [10, 5], - "OFF_THE_COAST" : [10, 6], - "CORNFIELD" : [11, 1], - "UNDERGROUND" : [11, 2], - "SANCTUARY" : [12, 1], - "RAVINE" : [12, 2], - "FLOODED_VALLEY" : [12, 3], - "INFERNAL" : [12, 4], - "BLOODY_PUDDLES" : [12, 5], - "WORKSHOP" : [12, 6], - "QUAD" : [13, 1], - "DARK_CASTLE" : [13, 2], - "MUDDY_PUDDLES" : [13, 3], - "OUCH" : [13, 4] +# top left and bottom right points of map page, used to split evenly into 3x2 grid +map_selection = { + "MAPS_TOP_LEFT" : [0.1698886936145284, 0.10729166666666666], + "MAPS_BOTTOM_RIGHT" : [0.8289396602226128, 0.6802083333333333], + "MAPS_TEXT_HEIGHT" : 0.025, + "MAP_NAME_FUZZ_THRESHOLD" : 70, } upgrade_keybinds = { diff --git a/src/statictest.py b/src/statictest.py new file mode 100644 index 0000000..86fc457 --- /dev/null +++ b/src/statictest.py @@ -0,0 +1,82 @@ +# Ordered maps for debug tests, not required for functionality +# Based on version 41.2.7621. Newer or older versions may have a different list of maps +expected_maps = [ + "MONKEY_MEADOW", + "IN_THE_LOOP", + "MIDDLE_OF_THE_ROAD", + "TREE_STUMP", + "TOWN_CENTER", + "ONE_TWO_TREE", + "SCRAPYARD", + "THE_CABIN", + "RESORT", + "SKATES", + "LOTUS_ISLAND", + "CANDY_FALLS", + "WINTER_PARK", + "CARVED", + "PARK_PATH", + "ALPINE_RUN", + "FROZEN_OVER", + "CUBISM", + "FOUR_CIRCLES", + "HEDGE", + "END_OF_THE_ROAD", + "LOGS", + "", + "", + "SULFUR_SPRINGS", + "WATER_PARK", + "POLYPHEMUS", + "COVERED_GARDEN", + "QUARRY", + "QUIET_STREET", + "BLOONARIUS_PRIME", + "BALANCE", + "ENCRYPTED", + "BAZAAR", + "ADORAS_TEMPLE", + "SPRING_SPRING", + "KARTSNDARTS", + "MOON_LANDING", + "HAUNTED", + "DOWNSTREAM", + "FIRING_RANGE", + "CRACKED", + "STREAMBED", + "CHUTES", + "RAKE", + "SPICE_ISLANDS", + "", + "", + "DARK_PATH", + "EROSION", + "MIDNIGHT_MANSION", + "SUNKEN_COLUMNS", + "X_FACTOR", + "MESA", + "GEARED", + "SPILLWAY", + "CARGO", + "PATS_POND", + "PENINSULA", + "HIGH_FINANCE", + "ANOTHER_BRICK", + "OFF_THE_COAST", + "CORNFIELD", + "UNDERGROUND", + "", + "", + "GLACIAL_TRAIL", + "DARK_DUNGEONS", + "SANCTUARY", + "RAVINE", + "FLOODED_VALLEY", + "INFERNAL", + "BLOODY_PUDDLES", + "WORKSHOP", + "QUAD", + "DARK_CASTLE", + "MUDDY_PUDDLES", + "OUCH", +] \ No newline at end of file