Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pull Request Issue 81 Dynamic Map Selection #82

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Pull Request Issue 81 Dynamic Map Selection #82

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 95 additions & 6 deletions src/Bot.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
import simulatedinput
import monitor
import gameplan
import statictest
from fuzzywuzzy import fuzz
from logger import logger as log

# Definently fix this
Expand Down Expand Up @@ -481,21 +483,108 @@ def restart_level(self, won=True):

self.wait_for_loading() # wait for loading screen

def select_map(self):
map_page = static.maps[self.settings["MAP"]][0]
map_index = static.maps[self.settings["MAP"]][1]
def getMapNameAtIndex(self, index):
# Get the top left and bottom right points
top_left_scaled = monitor.scaling(static.map_selection["MAPS_TOP_LEFT"])
bottom_right_scaled = monitor.scaling(static.map_selection["MAPS_BOTTOM_RIGHT"])
# need tuple for monitor.scaling call but just care about scaling the first value
text_height_scaled = monitor.scaling([static.map_selection["MAPS_TEXT_HEIGHT"], 0])[0]

# Setting up screen capture area using top left and bottom right points. Assume evenly spaced 3 x 2 grid of map options
screenshot_dimensions = {
'top': int(top_left_scaled[1] + (((bottom_right_scaled[1] - top_left_scaled[1]) / 2) * int(index / 3))),
'left': int(top_left_scaled[0] + (((bottom_right_scaled[0] - top_left_scaled[0]) / 3) * (index % 3))),
'width': int((bottom_right_scaled[0] - top_left_scaled[0]) / 3),
'height': int(text_height_scaled),
}

# Take Screenshot and get text
with mss.mss() as screenshotter:
screenshot = screenshotter.grab(screenshot_dimensions)
found_text, _ocrImage = ocr.getTextFromImage(screenshot)

if self.DEBUG:
from cv2 import imwrite, IMWRITE_PNG_COMPRESSION
def get_valid_filename(s):
s = str(s).strip().replace(' ', '_')
return re.sub(r'(?u)[^-\w.]', '', s)
imwrite(f"./DEBUG/OCR_DONE_FOUND_{get_valid_filename(found_text)}_{str(time.time())}.png", _ocrImage, [IMWRITE_PNG_COMPRESSION, 0])

return found_text

def select_map(self):
time.sleep(1)

simulatedinput.click("HOME_MENU_START")
simulatedinput.click("EXPERT_SELECTION", timeout=0.25)

simulatedinput.click("BEGINNER_SELECTION") # goto first page

# click to the right page
simulatedinput.click("RIGHT_ARROW_SELECTION", amount=(map_page - 1), timeout=0.1)
# save off the first map name to check for wrap around
initial_first_map_name = self.getMapNameAtIndex(0)
current_first_map_name = None # initialize to None so we don't hit break logic on the first loop
map_index_counter = 0
# tuple with map index and fuzz score
highest_fuzz_score_found = (0, 0)

# used for debug
map_names_extracted_correctly = 0
# tuple with map index and fuzz score
highest_fuzz_score_found_in_expected = (0, 0)

# format map selection setting by removing all spaces and underscores
formatted_map_selection_setting = self.settings["MAP"].replace("_", "").replace(" ", "").upper()
log.debug(f"formatted_map_selection_setting is \"{formatted_map_selection_setting}\"")

# interate through maps searching for the text. Exit if we get back to the first page or find the map setting
while True:
# if we get back to the first page break out of the loop
if current_first_map_name == initial_first_map_name:
break
# check each map in 3x2 grid
for i in range(6):
map_name = self.getMapNameAtIndex(i)
formatted_map_name = map_name.upper()

if self.DEBUG:
# only do debug if we have an expected maps at the index
if map_index_counter < len(statictest.expected_maps):
log.debug(f"Found map name \"{map_name}\" at index {i}")

# loop through all the expected maps checking making sure the highest fuzz score is at the expected index
for expected_map_index in range(len(statictest.expected_maps)):
formatted_expected_map_name = statictest.expected_maps[expected_map_index].replace("_", "").upper()
fuzz_score = fuzz.partial_ratio(formatted_map_name, formatted_expected_map_name)
# keep track of the index of the highest fuzz score found in the expected maps to compare
if fuzz_score > highest_fuzz_score_found_in_expected[1]:
highest_fuzz_score_found_in_expected = (expected_map_index, fuzz_score)

if highest_fuzz_score_found_in_expected[0] == map_index_counter:
log.debug(f"Correctly identified map at index {map_index_counter}")
map_names_extracted_correctly += 1
else:
log.debug(f"Highest fuzz score was not on the expected map")

fuzz_score = fuzz.partial_ratio(formatted_map_name, formatted_map_selection_setting)
if fuzz_score > highest_fuzz_score_found[1]:
highest_fuzz_score_found = (map_index_counter, fuzz_score)

map_index_counter += 1

# click the arrow once
simulatedinput.click("RIGHT_ARROW_SELECTION", amount=1, timeout=0.1)
current_first_map_name = self.getMapNameAtIndex(0)

if self.DEBUG:
log.debug(f"Found {map_names_extracted_correctly}/{map_index_counter} map names correctly")

# If we didn't find the map name on any of the pages that match above the threshold then log an error and return
if highest_fuzz_score_found[1] < static.map_selection["MAP_NAME_FUZZ_THRESHOLD"]:
raise Exception(f"Unable to find specified map above threshold {highest_fuzz_score_found[1]} < {static.map_selection['MAP_NAME_FUZZ_THRESHOLD']}")

simulatedinput.click("MAP_INDEX_" + str(map_index)) # Click correct map
# click the arrow once
simulatedinput.click("RIGHT_ARROW_SELECTION", amount=int(highest_fuzz_score_found[0] / 6), timeout=0.1)
simulatedinput.click("MAP_INDEX_" + str(int(highest_fuzz_score_found[0] % 6) + 1)) # Click correct map

if self.SANDBOX:
simulatedinput.click("EASY_MODE") # Select Difficulty
Expand Down
63 changes: 6 additions & 57 deletions src/ocr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,62 +13,11 @@

def formatImageOCR(originalScreenshot):
screenshot = np.array(originalScreenshot, dtype=np.uint8)
# Get local maximum:
kernelSize = 5
maxKernel = cv2.getStructuringElement(cv2.MORPH_RECT, (kernelSize, kernelSize))
localMax = cv2.morphologyEx(screenshot, cv2.MORPH_CLOSE, maxKernel, None, None, 1, cv2.BORDER_REFLECT101)
# Perform gain division
# print(screenshot, localMax)
gainDivision = np.where(localMax == 0, 0, (screenshot / localMax))
# Clip the values to [0,255]
gainDivision = np.clip((255 * gainDivision), 0, 255)
# Convert the mat type from float to uint8:
gainDivision = gainDivision.astype("uint8")
# Convert RGB to grayscale:
grayscaleImage = cv2.cvtColor(gainDivision, cv2.COLOR_BGR2GRAY)
# Resize image to improve the quality
grayscaleImage = cv2.resize(grayscaleImage,(0,0), fx=3.0, fy=3.0)
# Get binary image via Otsu:
_, final_image = cv2.threshold(grayscaleImage, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
# cv2.imwrite(f"./DEBUG/OCR_FORMAT_BINARY_IMAGE_{str(time.time())}.png", final_image, [cv2.IMWRITE_PNG_COMPRESSION, 0])

# Set kernel (structuring element) size:
kernelSize = 3
# Set morph operation iterations:
opIterations = 1
# Get the structuring element:
morphKernel = cv2.getStructuringElement(cv2.MORPH_RECT, (kernelSize, kernelSize))
# Perform closing:
final_image = cv2.morphologyEx( final_image, cv2.MORPH_CLOSE, morphKernel, None, None, opIterations, cv2.BORDER_REFLECT101 )
# cv2.imwrite(f"./DEBUG/OCR_FORMAT_BEFORE_FLOOD_{str(time.time())}.png", final_image, [cv2.IMWRITE_PNG_COMPRESSION, 0])

# Flood fill (white + black):
cv2.floodFill(final_image, mask=None, seedPoint=(int(0), int(0)), newVal=(255))
# Invert image so target blobs are colored in white:
final_image = 255 - final_image
# Find the blobs on the binary image:
contours, hierarchy = cv2.findContours(final_image, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
# Process the contours:
for i, c in enumerate(contours):
# Get contour hierarchy:
currentHierarchy = hierarchy[0][i][3]
# Look only for children contours (the holes):
if currentHierarchy != -1:
# Get the contour bounding rectangle:
boundRect = cv2.boundingRect(c)
# Get the dimensions of the bounding rect:
rectX = boundRect[0]
rectY = boundRect[1]
rectWidth = boundRect[2]
rectHeight = boundRect[3]
# Get the center of the contour the will act as
# seed point to the Flood-Filling:
fx = rectX + 0.5 * rectWidth
fy = rectY + 0.5 * rectHeight
# Fill the hole:
cv2.floodFill(final_image, mask=None, seedPoint=(int(fx), int(fy)), newVal=(0))
# cv2.imwrite(f"./DEBUG/OCR_FLOOD_{i}_{str(time.time())}.png", final_image, [cv2.IMWRITE_PNG_COMPRESSION, 0])

grayscaleImage = cv2.cvtColor(screenshot, cv2.COLOR_BGR2GRAY)
binary = cv2.threshold(grayscaleImage, 245, 255, cv2.THRESH_BINARY)[1]
binary = cv2.bitwise_not(binary)
kernel = np.ones((2, 2), np.uint8)
final_image = cv2.dilate(binary, kernel, iterations=1)
return final_image


Expand All @@ -86,5 +35,5 @@ def getTextFromImage(image):

# NOTE: This part seems to be buggy
# Get current round from screenshot with tesseract
return pytesseract.image_to_string(imageCandidate, config='--psm 7').replace("\n", ""), imageCandidate
return pytesseract.image_to_string(imageCandidate, config='--psm 10 --oem 3 -c tessedit_char_whitelist=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789/').replace("\n", ""), imageCandidate

71 changes: 6 additions & 65 deletions src/static.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,71 +136,12 @@ class Map (Difficulty, Gamemode):
"HERO" : "u"
}

# "NAME" : [PAGE, INDEX]
maps = {
"MONKEY_MEADOW" : [1, 1],
"TREE_STUMP" : [1, 2],
"TOWN_CENTER" : [1, 3],
"SCRAPYARD" : [1, 4],
"THE_CABIN" : [1, 5],
"RESORT" : [1, 6],
"SKATES" : [2, 1],
"LOTUS_ISLAND" : [2, 2],
"CANDY_FALLS" : [2, 3],
"WINTER_PARK" : [2, 4],
"CARVED" : [2, 5],
"PARK_PATH" : [2, 6],
"ALPINE_RUN" : [3, 1],
"FROZEN_OVER" : [3, 2],
"IN_THE_LOOP" : [3, 3],
"CUBISM" : [3, 4],
"FOUR_CIRCLES" : [3, 5],
"HEDGE" : [3, 6],
"END_OF_THE_ROAD" : [4, 1],
"LOGS" : [4, 2],
"COVERED_GARDEN" : [5, 1],
"QUARRY" : [5, 2],
"QUIET_STREET" : [5, 3],
"BLOONARIUS_PRIME" : [5, 4],
"BALANCE" : [5, 5],
"ENCRYPTED" : [5, 6],
"BAZAAR" : [6, 1],
"ADORAS_TEMPLE" : [6, 2],
"SPRING_SPRING" : [6, 3],
"KARTSNDARTS" : [6, 4],
"MOON_LANDING" : [6, 5],
"HAUNTED" : [6, 6],
"DOWNSTREAM" : [7, 1],
"FIRING_RANGE" : [7, 2],
"CRACKED" : [7, 3],
"STREAMBED" : [7, 4],
"CHUTES" : [7, 5],
"RAKE" : [7, 6],
"SPICE_ISLANDS" : [8, 1],
"MIDNIGHT_MANSION" : [9, 1],
"SUNKEN_COLUMNS" : [9, 2],
"XFACTOR" : [9, 3],
"MESA" : [9, 4],
"GEARED" : [9, 5],
"SPILLWAY" : [9, 6],
"CARGO" : [10, 1],
"PATS_POND" : [10, 2],
"PENINSULA" : [10, 3],
"HIGH_FINANCE" : [10, 4],
"ANOTHER_BRICK" : [10, 5],
"OFF_THE_COAST" : [10, 6],
"CORNFIELD" : [11, 1],
"UNDERGROUND" : [11, 2],
"SANCTUARY" : [12, 1],
"RAVINE" : [12, 2],
"FLOODED_VALLEY" : [12, 3],
"INFERNAL" : [12, 4],
"BLOODY_PUDDLES" : [12, 5],
"WORKSHOP" : [12, 6],
"QUAD" : [13, 1],
"DARK_CASTLE" : [13, 2],
"MUDDY_PUDDLES" : [13, 3],
"OUCH" : [13, 4]
# top left and bottom right points of map page, used to split evenly into 3x2 grid
map_selection = {
"MAPS_TOP_LEFT" : [0.1698886936145284, 0.10729166666666666],
"MAPS_BOTTOM_RIGHT" : [0.8289396602226128, 0.6802083333333333],
"MAPS_TEXT_HEIGHT" : 0.025,
"MAP_NAME_FUZZ_THRESHOLD" : 70,
}

upgrade_keybinds = {
Expand Down
82 changes: 82 additions & 0 deletions src/statictest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# Ordered maps for debug tests, not required for functionality
# Based on version 41.2.7621. Newer or older versions may have a different list of maps
expected_maps = [
"MONKEY_MEADOW",
"IN_THE_LOOP",
"MIDDLE_OF_THE_ROAD",
"TREE_STUMP",
"TOWN_CENTER",
"ONE_TWO_TREE",
"SCRAPYARD",
"THE_CABIN",
"RESORT",
"SKATES",
"LOTUS_ISLAND",
"CANDY_FALLS",
"WINTER_PARK",
"CARVED",
"PARK_PATH",
"ALPINE_RUN",
"FROZEN_OVER",
"CUBISM",
"FOUR_CIRCLES",
"HEDGE",
"END_OF_THE_ROAD",
"LOGS",
"",
"",
"SULFUR_SPRINGS",
"WATER_PARK",
"POLYPHEMUS",
"COVERED_GARDEN",
"QUARRY",
"QUIET_STREET",
"BLOONARIUS_PRIME",
"BALANCE",
"ENCRYPTED",
"BAZAAR",
"ADORAS_TEMPLE",
"SPRING_SPRING",
"KARTSNDARTS",
"MOON_LANDING",
"HAUNTED",
"DOWNSTREAM",
"FIRING_RANGE",
"CRACKED",
"STREAMBED",
"CHUTES",
"RAKE",
"SPICE_ISLANDS",
"",
"",
"DARK_PATH",
"EROSION",
"MIDNIGHT_MANSION",
"SUNKEN_COLUMNS",
"X_FACTOR",
"MESA",
"GEARED",
"SPILLWAY",
"CARGO",
"PATS_POND",
"PENINSULA",
"HIGH_FINANCE",
"ANOTHER_BRICK",
"OFF_THE_COAST",
"CORNFIELD",
"UNDERGROUND",
"",
"",
"GLACIAL_TRAIL",
"DARK_DUNGEONS",
"SANCTUARY",
"RAVINE",
"FLOODED_VALLEY",
"INFERNAL",
"BLOODY_PUDDLES",
"WORKSHOP",
"QUAD",
"DARK_CASTLE",
"MUDDY_PUDDLES",
"OUCH",
]