diff --git a/src/Bot.py b/src/Bot.py index 2f61702..dd20652 100644 --- a/src/Bot.py +++ b/src/Bot.py @@ -14,6 +14,8 @@ import simulatedinput import monitor import gameplan +import statictest +from fuzzywuzzy import fuzz from logger import logger as log # Definently fix this @@ -481,10 +483,36 @@ def restart_level(self, won=True): self.wait_for_loading() # wait for loading screen - def select_map(self): - map_page = static.maps[self.settings["MAP"]][0] - map_index = static.maps[self.settings["MAP"]][1] + def getMapNameAtIndex(self, index): + # Get the top left and bottom right points + top_left_scaled = monitor.scaling(static.map_selection["MAPS_TOP_LEFT"]) + bottom_right_scaled = monitor.scaling(static.map_selection["MAPS_BOTTOM_RIGHT"]) + # need tuple for monitor.scaling call but just care about scaling the first value + text_height_scaled = monitor.scaling([static.map_selection["MAPS_TEXT_HEIGHT"], 0])[0] + # Setting up screen capture area using top left and bottom right points. Assume evenly spaced 3 x 2 grid of map options + screenshot_dimensions = { + 'top': int(top_left_scaled[1] + (((bottom_right_scaled[1] - top_left_scaled[1]) / 2) * int(index / 3))), + 'left': int(top_left_scaled[0] + (((bottom_right_scaled[0] - top_left_scaled[0]) / 3) * (index % 3))), + 'width': int((bottom_right_scaled[0] - top_left_scaled[0]) / 3), + 'height': int(text_height_scaled), + } + + # Take Screenshot and get text + with mss.mss() as screenshotter: + screenshot = screenshotter.grab(screenshot_dimensions) + found_text, _ocrImage = ocr.getTextFromImage(screenshot) + + if self.DEBUG: + from cv2 import imwrite, IMWRITE_PNG_COMPRESSION + def get_valid_filename(s): + s = str(s).strip().replace(' ', '_') + return re.sub(r'(?u)[^-\w.]', '', s) + imwrite(f"./DEBUG/OCR_DONE_FOUND_{get_valid_filename(found_text)}_{str(time.time())}.png", _ocrImage, [IMWRITE_PNG_COMPRESSION, 0]) + + return found_text + + def select_map(self): time.sleep(1) simulatedinput.click("HOME_MENU_START") @@ -492,10 +520,71 @@ def select_map(self): simulatedinput.click("BEGINNER_SELECTION") # goto first page - # click to the right page - simulatedinput.click("RIGHT_ARROW_SELECTION", amount=(map_page - 1), timeout=0.1) + # save off the first map name to check for wrap around + initial_first_map_name = self.getMapNameAtIndex(0) + current_first_map_name = None # initialize to None so we don't hit break logic on the first loop + map_index_counter = 0 + # tuple with map index and fuzz score + highest_fuzz_score_found = (0, 0) + + # used for debug + map_names_extracted_correctly = 0 + # tuple with map index and fuzz score + highest_fuzz_score_found_in_expected = (0, 0) + + # format map selection setting by removing all spaces and underscores + formatted_map_selection_setting = self.settings["MAP"].replace("_", "").replace(" ", "").upper() + log.debug(f"formatted_map_selection_setting is \"{formatted_map_selection_setting}\"") + + # interate through maps searching for the text. Exit if we get back to the first page or find the map setting + while True: + # if we get back to the first page break out of the loop + if current_first_map_name == initial_first_map_name: + break + # check each map in 3x2 grid + for i in range(6): + map_name = self.getMapNameAtIndex(i) + formatted_map_name = map_name.upper() + + if self.DEBUG: + # only do debug if we have an expected maps at the index + if map_index_counter < len(statictest.expected_maps): + log.debug(f"Found map name \"{map_name}\" at index {i}") + + # loop through all the expected maps checking making sure the highest fuzz score is at the expected index + for expected_map_index in range(len(statictest.expected_maps)): + formatted_expected_map_name = statictest.expected_maps[expected_map_index].replace("_", "").upper() + fuzz_score = fuzz.partial_ratio(formatted_map_name, formatted_expected_map_name) + # keep track of the index of the highest fuzz score found in the expected maps to compare + if fuzz_score > highest_fuzz_score_found_in_expected[1]: + highest_fuzz_score_found_in_expected = (expected_map_index, fuzz_score) + + if highest_fuzz_score_found_in_expected[0] == map_index_counter: + log.debug(f"Correctly identified map at index {map_index_counter}") + map_names_extracted_correctly += 1 + else: + log.debug(f"Highest fuzz score was not on the expected map") + + fuzz_score = fuzz.partial_ratio(formatted_map_name, formatted_map_selection_setting) + if fuzz_score > highest_fuzz_score_found[1]: + highest_fuzz_score_found = (map_index_counter, fuzz_score) + + map_index_counter += 1 + + # click the arrow once + simulatedinput.click("RIGHT_ARROW_SELECTION", amount=1, timeout=0.1) + current_first_map_name = self.getMapNameAtIndex(0) + + if self.DEBUG: + log.debug(f"Found {map_names_extracted_correctly}/{map_index_counter} map names correctly") + + # If we didn't find the map name on any of the pages that match above the threshold then log an error and return + if highest_fuzz_score_found[1] < static.map_selection["MAP_NAME_FUZZ_THRESHOLD"]: + raise Exception(f"Unable to find specified map above threshold {highest_fuzz_score_found[1]} < {static.map_selection['MAP_NAME_FUZZ_THRESHOLD']}") - simulatedinput.click("MAP_INDEX_" + str(map_index)) # Click correct map + # click the arrow once + simulatedinput.click("RIGHT_ARROW_SELECTION", amount=int(highest_fuzz_score_found[0] / 6), timeout=0.1) + simulatedinput.click("MAP_INDEX_" + str(int(highest_fuzz_score_found[0] % 6) + 1)) # Click correct map if self.SANDBOX: simulatedinput.click("EASY_MODE") # Select Difficulty diff --git a/src/ocr.py b/src/ocr.py index e25d4b4..24e09c4 100644 --- a/src/ocr.py +++ b/src/ocr.py @@ -13,62 +13,11 @@ def formatImageOCR(originalScreenshot): screenshot = np.array(originalScreenshot, dtype=np.uint8) - # Get local maximum: - kernelSize = 5 - maxKernel = cv2.getStructuringElement(cv2.MORPH_RECT, (kernelSize, kernelSize)) - localMax = cv2.morphologyEx(screenshot, cv2.MORPH_CLOSE, maxKernel, None, None, 1, cv2.BORDER_REFLECT101) - # Perform gain division - # print(screenshot, localMax) - gainDivision = np.where(localMax == 0, 0, (screenshot / localMax)) - # Clip the values to [0,255] - gainDivision = np.clip((255 * gainDivision), 0, 255) - # Convert the mat type from float to uint8: - gainDivision = gainDivision.astype("uint8") - # Convert RGB to grayscale: - grayscaleImage = cv2.cvtColor(gainDivision, cv2.COLOR_BGR2GRAY) - # Resize image to improve the quality - grayscaleImage = cv2.resize(grayscaleImage,(0,0), fx=3.0, fy=3.0) - # Get binary image via Otsu: - _, final_image = cv2.threshold(grayscaleImage, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) - # cv2.imwrite(f"./DEBUG/OCR_FORMAT_BINARY_IMAGE_{str(time.time())}.png", final_image, [cv2.IMWRITE_PNG_COMPRESSION, 0]) - - # Set kernel (structuring element) size: - kernelSize = 3 - # Set morph operation iterations: - opIterations = 1 - # Get the structuring element: - morphKernel = cv2.getStructuringElement(cv2.MORPH_RECT, (kernelSize, kernelSize)) - # Perform closing: - final_image = cv2.morphologyEx( final_image, cv2.MORPH_CLOSE, morphKernel, None, None, opIterations, cv2.BORDER_REFLECT101 ) - # cv2.imwrite(f"./DEBUG/OCR_FORMAT_BEFORE_FLOOD_{str(time.time())}.png", final_image, [cv2.IMWRITE_PNG_COMPRESSION, 0]) - - # Flood fill (white + black): - cv2.floodFill(final_image, mask=None, seedPoint=(int(0), int(0)), newVal=(255)) - # Invert image so target blobs are colored in white: - final_image = 255 - final_image - # Find the blobs on the binary image: - contours, hierarchy = cv2.findContours(final_image, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) - # Process the contours: - for i, c in enumerate(contours): - # Get contour hierarchy: - currentHierarchy = hierarchy[0][i][3] - # Look only for children contours (the holes): - if currentHierarchy != -1: - # Get the contour bounding rectangle: - boundRect = cv2.boundingRect(c) - # Get the dimensions of the bounding rect: - rectX = boundRect[0] - rectY = boundRect[1] - rectWidth = boundRect[2] - rectHeight = boundRect[3] - # Get the center of the contour the will act as - # seed point to the Flood-Filling: - fx = rectX + 0.5 * rectWidth - fy = rectY + 0.5 * rectHeight - # Fill the hole: - cv2.floodFill(final_image, mask=None, seedPoint=(int(fx), int(fy)), newVal=(0)) - # cv2.imwrite(f"./DEBUG/OCR_FLOOD_{i}_{str(time.time())}.png", final_image, [cv2.IMWRITE_PNG_COMPRESSION, 0]) - + grayscaleImage = cv2.cvtColor(screenshot, cv2.COLOR_BGR2GRAY) + binary = cv2.threshold(grayscaleImage, 245, 255, cv2.THRESH_BINARY)[1] + binary = cv2.bitwise_not(binary) + kernel = np.ones((2, 2), np.uint8) + final_image = cv2.dilate(binary, kernel, iterations=1) return final_image @@ -86,5 +35,5 @@ def getTextFromImage(image): # NOTE: This part seems to be buggy # Get current round from screenshot with tesseract - return pytesseract.image_to_string(imageCandidate, config='--psm 7').replace("\n", ""), imageCandidate + return pytesseract.image_to_string(imageCandidate, config='--psm 10 --oem 3 -c tessedit_char_whitelist=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789/').replace("\n", ""), imageCandidate diff --git a/src/static.py b/src/static.py index 8d3fbe0..bf2ac76 100644 --- a/src/static.py +++ b/src/static.py @@ -136,71 +136,12 @@ class Map (Difficulty, Gamemode): "HERO" : "u" } -# "NAME" : [PAGE, INDEX] -maps = { - "MONKEY_MEADOW" : [1, 1], - "TREE_STUMP" : [1, 2], - "TOWN_CENTER" : [1, 3], - "SCRAPYARD" : [1, 4], - "THE_CABIN" : [1, 5], - "RESORT" : [1, 6], - "SKATES" : [2, 1], - "LOTUS_ISLAND" : [2, 2], - "CANDY_FALLS" : [2, 3], - "WINTER_PARK" : [2, 4], - "CARVED" : [2, 5], - "PARK_PATH" : [2, 6], - "ALPINE_RUN" : [3, 1], - "FROZEN_OVER" : [3, 2], - "IN_THE_LOOP" : [3, 3], - "CUBISM" : [3, 4], - "FOUR_CIRCLES" : [3, 5], - "HEDGE" : [3, 6], - "END_OF_THE_ROAD" : [4, 1], - "LOGS" : [4, 2], - "COVERED_GARDEN" : [5, 1], - "QUARRY" : [5, 2], - "QUIET_STREET" : [5, 3], - "BLOONARIUS_PRIME" : [5, 4], - "BALANCE" : [5, 5], - "ENCRYPTED" : [5, 6], - "BAZAAR" : [6, 1], - "ADORAS_TEMPLE" : [6, 2], - "SPRING_SPRING" : [6, 3], - "KARTSNDARTS" : [6, 4], - "MOON_LANDING" : [6, 5], - "HAUNTED" : [6, 6], - "DOWNSTREAM" : [7, 1], - "FIRING_RANGE" : [7, 2], - "CRACKED" : [7, 3], - "STREAMBED" : [7, 4], - "CHUTES" : [7, 5], - "RAKE" : [7, 6], - "SPICE_ISLANDS" : [8, 1], - "MIDNIGHT_MANSION" : [9, 1], - "SUNKEN_COLUMNS" : [9, 2], - "XFACTOR" : [9, 3], - "MESA" : [9, 4], - "GEARED" : [9, 5], - "SPILLWAY" : [9, 6], - "CARGO" : [10, 1], - "PATS_POND" : [10, 2], - "PENINSULA" : [10, 3], - "HIGH_FINANCE" : [10, 4], - "ANOTHER_BRICK" : [10, 5], - "OFF_THE_COAST" : [10, 6], - "CORNFIELD" : [11, 1], - "UNDERGROUND" : [11, 2], - "SANCTUARY" : [12, 1], - "RAVINE" : [12, 2], - "FLOODED_VALLEY" : [12, 3], - "INFERNAL" : [12, 4], - "BLOODY_PUDDLES" : [12, 5], - "WORKSHOP" : [12, 6], - "QUAD" : [13, 1], - "DARK_CASTLE" : [13, 2], - "MUDDY_PUDDLES" : [13, 3], - "OUCH" : [13, 4] +# top left and bottom right points of map page, used to split evenly into 3x2 grid +map_selection = { + "MAPS_TOP_LEFT" : [0.1698886936145284, 0.10729166666666666], + "MAPS_BOTTOM_RIGHT" : [0.8289396602226128, 0.6802083333333333], + "MAPS_TEXT_HEIGHT" : 0.025, + "MAP_NAME_FUZZ_THRESHOLD" : 70, } upgrade_keybinds = { diff --git a/src/statictest.py b/src/statictest.py new file mode 100644 index 0000000..86fc457 --- /dev/null +++ b/src/statictest.py @@ -0,0 +1,82 @@ +# Ordered maps for debug tests, not required for functionality +# Based on version 41.2.7621. Newer or older versions may have a different list of maps +expected_maps = [ + "MONKEY_MEADOW", + "IN_THE_LOOP", + "MIDDLE_OF_THE_ROAD", + "TREE_STUMP", + "TOWN_CENTER", + "ONE_TWO_TREE", + "SCRAPYARD", + "THE_CABIN", + "RESORT", + "SKATES", + "LOTUS_ISLAND", + "CANDY_FALLS", + "WINTER_PARK", + "CARVED", + "PARK_PATH", + "ALPINE_RUN", + "FROZEN_OVER", + "CUBISM", + "FOUR_CIRCLES", + "HEDGE", + "END_OF_THE_ROAD", + "LOGS", + "", + "", + "SULFUR_SPRINGS", + "WATER_PARK", + "POLYPHEMUS", + "COVERED_GARDEN", + "QUARRY", + "QUIET_STREET", + "BLOONARIUS_PRIME", + "BALANCE", + "ENCRYPTED", + "BAZAAR", + "ADORAS_TEMPLE", + "SPRING_SPRING", + "KARTSNDARTS", + "MOON_LANDING", + "HAUNTED", + "DOWNSTREAM", + "FIRING_RANGE", + "CRACKED", + "STREAMBED", + "CHUTES", + "RAKE", + "SPICE_ISLANDS", + "", + "", + "DARK_PATH", + "EROSION", + "MIDNIGHT_MANSION", + "SUNKEN_COLUMNS", + "X_FACTOR", + "MESA", + "GEARED", + "SPILLWAY", + "CARGO", + "PATS_POND", + "PENINSULA", + "HIGH_FINANCE", + "ANOTHER_BRICK", + "OFF_THE_COAST", + "CORNFIELD", + "UNDERGROUND", + "", + "", + "GLACIAL_TRAIL", + "DARK_DUNGEONS", + "SANCTUARY", + "RAVINE", + "FLOODED_VALLEY", + "INFERNAL", + "BLOODY_PUDDLES", + "WORKSHOP", + "QUAD", + "DARK_CASTLE", + "MUDDY_PUDDLES", + "OUCH", +] \ No newline at end of file