alpcentaur/fdb-spider
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

changed size of virtual window, added some scrolling and shortened the time for js lazy loading enforced slow downloading

Browse source
This commit is contained in:
alpcentaur 2024-02-11 17:08:33 +00:00
parent a56569712e
commit 7ba196b0c2
2 changed files with 10 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
spiders/__pycache__/fdb_spider.cpython-311.pyc
View file

Binary file not shown.

15
spiders/fdb_spider.py
View file

@ -148,7 +148,8 @@ class fdb_spider(object):
from selenium.webdriver.chrome.service import Service from selenium.webdriver.chrome.service import Service
#from selenium.webdriver.common.action_chains import ActionChains #from selenium.webdriver.common.action_chains import ActionChains
from pyvirtualdisplay import Display from pyvirtualdisplay import Display
display = Display(visible=0, size=(800, 800)) # changed display to 1200, because element was not found in "mobile version" with 800 width
display = Display(visible=0, size=(1200, 800))
display.start() display.start()
##outputdir = '.' ##outputdir = '.'
@ -157,19 +158,23 @@ class fdb_spider(object):
##driver = webdriver.Chrome('/usr/bin/chromium') ##driver = webdriver.Chrome('/usr/bin/chromium')
options = webdriver.ChromeOptions() options = webdriver.ChromeOptions()
options.add_argument('headless') #options.add_argument('headless')
options.add_argument("--remote-debugging-port=9222") options.add_argument("--remote-debugging-port=9222")
options.add_argument('--no-sandbox') options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage') options.add_argument('--disable-dev-shm-usage')
service = Service(executable_path='/usr/bin/chromedriver') service = Service(executable_path='/usr/bin/chromedriver')
driver = webdriver.Chrome(options=options, service=service) driver = webdriver.Chrome(options=options, service=service)
# driver = webdriver.Chrome() # driver = webdriver.Chrome()
driver.implicitly_wait(10) driver.implicitly_wait(5)
driver.get(entry_jsdomain) driver.get(entry_jsdomain)
for i in range(len(entry_jsiteration_var_list)): for i in range(len(entry_jsiteration_var_list)):
time.sleep(2) time.sleep(1)
print('trying to get element') print('trying to get element')
try: try:
# scroll down, to get the javascript view loading to get the elements
driver.execute_script("scroll(0, 600)")
element = driver.find_element( element = driver.find_element(
"xpath", "xpath",
entry_list_jslink1 entry_list_jslink1
@ -186,7 +191,7 @@ class fdb_spider(object):
print('clicking..') print('clicking..')
time.sleep(1) time.sleep(1)
element.click() element.click()
time.sleep(2) time.sleep(1)
#window_after = driver.window_handles[1] #window_after = driver.window_handles[1]
print('length of the window handles', len(driver.window_handles)) print('length of the window handles', len(driver.window_handles))
#driver.switch_to.window(window_after) #driver.switch_to.window(window_after)