1

I try to scrape products items details on the site below, but the script always fails with the error no such element, although the element is there. Can anyone can help to solve the issue? My code:

from time import sleep

from scrapy import Spider
from selenium import webdriver
from scrapy.selector import Selector
from scrapy.http import Request
from selenium.common.exceptions import NoSuchElementException
driver = webdriver.Chrome('D:\chromedriver_win32\chromedriver.exe')
driver.get('http://www.tesensors.com/global/en/product/inductive-capacitive/xs-xt-ref')
sleep(5)
#soemtime the site ask you select language and country so need click button as below
sign_in_button = driver.find_element_by_id('edit-submit--4')
sign_in_button.click()
sleep(5)
# scrapy content.total 1168 items, here there is no result.
product_model_name=driver.find_elements_by_xpath('span[@itemprop="name"]')
product_desc=driver.find_elements_by_xpath('span[@itemprop="description"]')
trotta
  • 1,232
  • 1
  • 16
  • 23
Yan Zhang
  • 341
  • 1
  • 6
  • 15

3 Answers3

1

Product data inside iframe

You can use an XPath to locate the :

iframe = driver.find_element_by_xpath("//iframe[@id='ecat']")

Then switch_to the :

driver.switch_to.frame(iframe)

Here's how to switch back to the default content (out of the ):

driver.switch_to.default_content()

Do not use time-sleep module, try explicit-waits.

see difference.

EX:

from scrapy import Spider
from selenium import webdriver
from scrapy.selector import Selector
from scrapy.http import Request
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC


driver = webdriver.Chrome('D:\chromedriver_win32\chromedriver.exe')
driver.get('http://www.tesensors.com/global/en/product/inductive-capacitive/xs-xt-ref')

#soemtime the site ask you select language and country so need click button as below
sign_in_button = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "edit-submit--4")))
sign_in_button.click()

#switch iframe
iframe = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//iframe[@id='ecat']")))
driver.switch_to.frame(iframe)

# scrapy content.total 1168 items, here there is no result.
product_model_name = driver.find_elements_by_xpath('//span[@itemprop="name"]')
print(product_model_name[0].text)

product_desc=driver.find_elements_by_xpath('//span[@itemprop="description"]')

print(product_model_name[0].text)
bharatk
  • 4,202
  • 5
  • 16
  • 30
0
import time
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By



driver = webdriver.Chrome(r"C:\Users\path\Desktop\chromedriver\chromedriver.exe")
driver.get('http://www.tesensors.com/global/en/product/inductive-capacitive/xs-xt-ref')

try:
    element = WebDriverWait(driver, 5).until(EC.presence_of_element_located((By.ID, "edit-submit--4")))
    element.submit()
except:
    print("proceeding further")

iframe = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//iframe[@id='ecat']")))
driver.switch_to.frame(iframe)
product_model_name = driver.find_elements_by_xpath("//*[@itemprop='name']")
product_model_description = driver.find_elements_by_xpath("//*[@itemprop='description']")
names = []
description = []
for i in product_model_name:
    print(i.text)
    names.append(i.text)
for i in product_model_description:
    print(i.text)
    description.append(i.text)
G1Rao
  • 424
  • 5
  • 11
0

I used this method for getting elements:

from scrapy import Spider
import os
from selenium import webdriver
import time
from scrapy.selector import Selector
from scrapy.http import Request
from selenium.common.exceptions import NoSuchElementException
chromedriver = pathToDriver + 'chromedriver'
os.environ["webdriver.chrome.driver"] = chromedriver
driver = webdriver.Chrome(chromedriver)
driver.get('http://www.tesensors.com/global/en/product/inductive-capacitive/xs-xt-ref')
time.sleep(3)
#soemtime the site ask you select language and country so need click button as below
sign_in_button = driver.find_element_by_id('edit-submit--4')
sign_in_button.click()
time.sleep(3)
iframe_src = driver.find_element_by_id('ecat').get_attribute("src")
print(iframe_src)
driver.get(iframe_src)
# scrapy content.total 1168 items, here there is no result.
product_model_names=driver.find_elements_by_class_name('boldLevel2')
product_names = list()
for element in product_model_names:
    product_names.append(element.text)
print(product_names)

product_desc=driver.find_elements_by_class_name('level1')
product_descptions = list()
for element in product_desc:
    product_descptions.append(element.text)
print(product_descptions)

driver.close()
Nitin
  • 246
  • 1
  • 7