Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix selenium webdriver #40

Open
wants to merge 6 commits into
base: patch-1
Choose a base branch
from
Open

Fix selenium webdriver #40

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7f9185a
change selector
yoolandes Oct 25, 2022
77f7352
disable shm
yoolandes Oct 25, 2022
0fc4ffe
fix multi pages
yoolandes Oct 25, 2022
d8b41a9
Update google_images_download.py
janvdp Feb 28, 2023
6b117da
Update google_images_download.py
janvdp Apr 24, 2023
2cb2cbb
Merge remote-tracking branch 'yoolandes/patch-1' into patch-1
ftapajos May 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 20 additions & 16 deletions google_images_download/google_images_download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
# Import Libraries
import sys
import selenium.common.exceptions
from selenium.webdriver.common.by import By

version = (3, 0)
cur_version = sys.version_info
Expand Down Expand Up @@ -254,18 +255,21 @@ def download_page(self, url):
def download_extended_page(self, url, chromedriver, browser):
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.service import Service

if sys.version_info[0] < 3:
reload(sys)
sys.setdefaultencoding('utf8')
options = webdriver.ChromeOptions()
options.add_argument('--no-sandbox')
options.add_argument("--headless")
options.add_argument('--disable-dev-shm-usage')

if browser == 'Firefox':
browser = webdriver.Firefox()
else:
try:
browser = webdriver.Chrome(chromedriver, chrome_options=options)
browser = webdriver.Chrome(service=Service(chromedriver), chrome_options=options)
except Exception as e:
print("Looks like we cannot locate the path the 'chromedriver' (use the '--chromedriver' "
"argument to specify the path to the executable.) or google chrome browser is not "
Expand All @@ -275,6 +279,16 @@ def download_extended_page(self, url, chromedriver, browser):

# Open the link
browser.get(url)

time.sleep(1)

# Bypass "Before you continue" if it appears
try:
browser.find_element(By.CSS_SELECTOR, "[aria-label='Accept all']").click()
time.sleep(1)
except selenium.common.exceptions.NoSuchElementException:
pass

browser.execute_script("""
(function(XHR){
"use strict";
Expand Down Expand Up @@ -302,26 +316,16 @@ def download_extended_page(self, url, chromedriver, browser):
XHR.prototype._data = [];
})(XMLHttpRequest);
""")

time.sleep(1)

# Bypass "Before you continue" if it appears
try:
browser.find_element_by_css_selector("[aria-label='Accept all']").click()
time.sleep(1)
except selenium.common.exceptions.NoSuchElementException:
pass

print("Getting you a lot of images. This may take a few moments...")

element = browser.find_element_by_tag_name("body")
element = browser.find_element(By.TAG_NAME, "body")
# Scroll down
for i in range(50):
element.send_keys(Keys.PAGE_DOWN)
time.sleep(0.3)

try:
browser.find_element_by_xpath('//input[@value="Show more results"]').click()
browser.find_element(By.XPATH, '//input[@value="Show more results"]').click()
for i in range(50):
element.send_keys(Keys.PAGE_DOWN)
time.sleep(0.3) # bot id protection
Expand Down Expand Up @@ -402,9 +406,9 @@ def get_all_tabs(self, page):
def format_object(self, object):
data = object[1]
main = data[3]
info = data[9]
info = data[25]
if info is None:
info = data[11]
info = data[23]
formatted_object = {}
try:
formatted_object['image_height'] = main[2]
Expand Down