Locate something on the screen using PyScreeze on Python

  • locateOnScreen(image, grayscale=False) – Returns (left, top, width, height) coordinate of first found instance of the image on the screen. Returns None if not found on the screen.
  • locateCenterOnScreen(image, grayscale=False) – Returns (x, y) coordinates of the center of the first found instance of the image on the screen. Returns None if not found on the screen.
  • locateAllOnScreen(image, grayscale=False) – Returns a generator that yields (left, top, width, height) tuples for where the image is found on the screen.
  • locate(needleImage, haystackImage, grayscale=False) – Returns (left, top, width, height) coordinate of first found instance of needleImage in haystackImage. Returns None if not found on the screen.
  • locateAll(needleImage, haystackImage, grayscale=False) – Returns a generator that yields (left, top, width, height) tuples for where needleImage is found in haystackImage.

 

import pyscreeze
button7location = pyscreeze.locateOnScreen('calc7key.png')
button7location
(1416, 562, 50, 41)
button7x, button7y = pyscreeze.center(button7location)
button7x, button7y
(1441, 582)
pyscreeze.click(button7x, button7y)  # clicks the center of where the 7 button was found
import pyscreeze
x, y = pyscreeze.locateCenterOnScreen('calc7key.png')
pyscreeze.click(x, y)
import pyscreeze
for pos in pyscreeze.locateAllOnScreen('someButton.png')
...   print(pos)
...
(1101, 252, 50, 50)
(59, 481, 50, 50)
(1395, 640, 50, 50)
(1838, 676, 50, 50)
list(pyscreeze.locateAllOnScreen('someButton.png'))
[(1101, 252, 50, 50), (59, 481, 50, 50), (1395, 640, 50, 50), (1838, 676, 50, 50)]
import pyscreeze
button7location = pyscreeze.locateOnScreen('calc7key.png', grayscale=True)
button7location
(1416, 562, 50, 41)

References
https://pypi.org/project/PyScreeze/

Template Matching using OpenCV Python

image_gray: np.ndarray
TM_METHOD=cv2.TM_CCOEFF_NORMED

def __init__(self):
    self.take_screenshot()

def take_screenshot(self):
    user32 = ctypes.windll.user32
    user32.SetProcessDPIAware()
    image = ImageGrab.grab()
    # save on file
    # image.save('screenshot.jpg', format='JPEG', quality=20)

    # load image from memory
    image_rgb = np.array(image)

    # load from file
    # image_rgb=cv2.imread('screenshot.jpg')
    self.image_gray = cv2.cvtColor(image_rgb, cv2.COLOR_BGR2GRAY)

@staticmethod
def save_screenshot():
    user32 = ctypes.windll.user32
    user32.SetProcessDPIAware()
    image = ImageGrab.grab()
    image.save("screenshot.jpg")

def template_available(self, template_path: str, threshold=0.9):

    # self.take_screenshot()

    template = cv2.imread(template_path, 0)
    # w, h = template.shape[::-1]

    res = cv2.matchTemplate(self.image_gray, template, self.TM_METHOD)
    loc = np.where(res >= threshold)

    if len(loc[0]) == 0 & len(loc[1]) == 0:
        return False
    else:
        print("template_available : " + template_path)
        return True

def template_count(self, template_path: str, threshold=0.9):

    # self.take_screenshot()

    template = cv2.imread(template_path, 0)
    # w, h = template.shape[::-1]

    res = cv2.matchTemplate(self.image_gray, template, self.TM_METHOD)
    loc = np.where(res >= threshold)

    return len(loc[0])

def template_location(self, template_path: str, threshold=0.9):

    self.take_screenshot()

    template = cv2.imread(template_path, 0)
    w, h = template.shape[::-1]

    res = cv2.matchTemplate(self.image_gray, template, self.TM_METHOD)
    loc = np.where(res >= threshold)

    if len(loc[0]) == 0 & len(loc[1]) == 0:
        return None
    else:
        click_y = loc[0][0] + h / 2
        click_x = loc[1][0] + w / 2

        return click_x, click_y

def move_mouse_to_template(self, template_path: str):
    loc = self.template_location(template_path)

    if loc is not None:
        click_x = loc[0]
        click_y = loc[1]
        pyautogui.moveTo(click_x, click_y)

def click_template(self, template_path: str, threshold=0.9):
    loc = self.template_location(template_path, threshold)

    if loc is not None:
        click_x = loc[0]
        click_y = loc[1]
        pyautogui.click(click_x, click_y)
        pyautogui.FAILSAFE = False
        pyautogui.moveTo(0, 0)
        print("click_template : " + template_path)
        return True

    return False

@staticmethod
def scroll_mouse(value=10):
    pyautogui.scroll(value)
    pyautogui.FAILSAFE = False
    pyautogui.moveTo(0, 0)

@staticmethod
def get_template_path(template: str):
    template_name = f"template{template}.jpg"
    template_path = os.path.join(os.getcwd(), 'templates', template_name)
    return template_path

 

Take Screenshot using Python PIL

# Importing Image and ImageGrab module from PIL package 
from PIL import Image, ImageGrab 
  
# creating an image object 
im1 = Image.open(r"C:\Users\sadow984\Desktop\download2.JPG") 
  
# using the grab method 
im2 = ImageGrab.grab(bbox = None) 
  
im2.show()

Taking screenshots of specific size or taking screenshots from specific region

# Importing Image and ImageGrab module from PIL package 
from PIL import Image, ImageGrab 
  
# creating an image object 
im1 = Image.open(r"C:\Users\sadow984\Desktop\download2.JPG") 
  
# using the grab method 
im2 = ImageGrab.grab(bbox =(0, 0, 300, 300)) 
  
im2.show()

Parameters: box – The crop rectangle, as a (left, upper, right, lower)-tuple.

References
https://www.geeksforgeeks.org/pyhton-pil-imagegrab-grab-method/
https://stackoverflow.com/questions/19697210/taking-screen-shots-of-specific-size
https://stackoverflow.com/questions/47337811/what-do-the-parameters-of-bbox-mean-in-pillow

numpy.where() in Python

returns the indices of elements in an input array where the given condition is satisfied.

# Python program explaining 
# where() function 

import numpy as np 

np.where([[True, False], [True, True]], 
    [[1, 2], [3, 4]], [[5, 6], [7, 8]]) 

# Python program explaining 
# where() function 

import numpy as np 

# a is an array of integers. 
a = np.array([[1, 2, 3], [4, 5, 6]]) 

print(a) 

print ('Indices of elements <4') 

b = np.where(a<4) 
print(b) 

print("Elements which are <4") 
print(a[b])

References
https://www.geeksforgeeks.org/numpy-where-in-python/

Querying data with Pandas .query() method

Single condition filtering

# importing pandas package 
import pandas as pd 

# making data frame from csv file 
data = pd.read_csv("employees.csv") 

# replacing blank spaces with '_' 
data.columns =[column.replace(" ", "_") for column in data.columns] 

# filtering with query method 
data.query('Senior_Management == True', inplace = True) 

# display 
data

Multiple condition filtering

# importing pandas package 
import pandas as pd 

# making data frame from csv file 
data = pd.read_csv("employees.csv") 

# replacing blank spaces with '_' 
data.columns =[column.replace(" ", "_") for column in data.columns] 

# filtering with query method 
data.query('Senior_Management == True
      and Gender =="Male" and Team =="Marketing"
      and First_Name =="Johnny"', inplace = True) 

# display 
data

References
https://www.geeksforgeeks.org/python-filtering-data-with-pandas-query-method/