changed size of virtual window, added some scrolling and shortened the time for js lazy loading enforced slow downloading

spiders/fdb_spider.py

@@ -148,7 +148,8 @@ class fdb_spider(object):
                         from selenium.webdriver.chrome.service import Service
                         #from selenium.webdriver.common.action_chains import ActionChains
                         from pyvirtualdisplay import Display
-                        display = Display(visible=0, size=(800, 800))  
+                        # changed display to 1200, because element was not found in "mobile version" with 800 width
+                        display = Display(visible=0, size=(1200, 800))  
                         display.start()
 
                         ##outputdir = '.'
@@ -157,19 +158,23 @@ class fdb_spider(object):
                         ##driver = webdriver.Chrome('/usr/bin/chromium')
                         
                         options = webdriver.ChromeOptions()
-                        options.add_argument('headless')
+                        #options.add_argument('headless')
                         options.add_argument("--remote-debugging-port=9222")
                         options.add_argument('--no-sandbox')
                         options.add_argument('--disable-dev-shm-usage')
                         service = Service(executable_path='/usr/bin/chromedriver')
                         driver = webdriver.Chrome(options=options, service=service)
                         # driver = webdriver.Chrome()
-                        driver.implicitly_wait(10) 
+                        driver.implicitly_wait(5) 
                         driver.get(entry_jsdomain)
                         for i in range(len(entry_jsiteration_var_list)):
-                            time.sleep(2)
+                            time.sleep(1)
                             print('trying to get element')
                             try:
+                                
+                                # scroll down, to get the javascript view loading to get the elements
+                                driver.execute_script("scroll(0, 600)")
+                                
                                 element = driver.find_element(
                                     "xpath",
                                     entry_list_jslink1
@@ -186,7 +191,7 @@ class fdb_spider(object):
                                 print('clicking..')
                                 time.sleep(1)
                                 element.click()
-                                time.sleep(2)
+                                time.sleep(1)
                                 #window_after = driver.window_handles[1]
                                 print('length of the window handles', len(driver.window_handles))
                                 #driver.switch_to.window(window_after)