diff --git a/spiders/__pycache__/fdb_spider.cpython-311.pyc b/spiders/__pycache__/fdb_spider.cpython-311.pyc index 549e0a4..2d81f66 100644 Binary files a/spiders/__pycache__/fdb_spider.cpython-311.pyc and b/spiders/__pycache__/fdb_spider.cpython-311.pyc differ diff --git a/spiders/fdb_spider.py b/spiders/fdb_spider.py index 5557951..ea63cd2 100644 --- a/spiders/fdb_spider.py +++ b/spiders/fdb_spider.py @@ -148,7 +148,8 @@ class fdb_spider(object): from selenium.webdriver.chrome.service import Service #from selenium.webdriver.common.action_chains import ActionChains from pyvirtualdisplay import Display - display = Display(visible=0, size=(800, 800)) + # changed display to 1200, because element was not found in "mobile version" with 800 width + display = Display(visible=0, size=(1200, 800)) display.start() ##outputdir = '.' @@ -157,19 +158,23 @@ class fdb_spider(object): ##driver = webdriver.Chrome('/usr/bin/chromium') options = webdriver.ChromeOptions() - options.add_argument('headless') + #options.add_argument('headless') options.add_argument("--remote-debugging-port=9222") options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') service = Service(executable_path='/usr/bin/chromedriver') driver = webdriver.Chrome(options=options, service=service) # driver = webdriver.Chrome() - driver.implicitly_wait(10) + driver.implicitly_wait(5) driver.get(entry_jsdomain) for i in range(len(entry_jsiteration_var_list)): - time.sleep(2) + time.sleep(1) print('trying to get element') try: + + # scroll down, to get the javascript view loading to get the elements + driver.execute_script("scroll(0, 600)") + element = driver.find_element( "xpath", entry_list_jslink1 @@ -186,7 +191,7 @@ class fdb_spider(object): print('clicking..') time.sleep(1) element.click() - time.sleep(2) + time.sleep(1) #window_after = driver.window_handles[1] print('length of the window handles', len(driver.window_handles)) #driver.switch_to.window(window_after)