use selenium in scrapy webdriver

https://pypi.python.org/pypi/selenium

from selenium import webdriver

from selenium.webdriver.common.keys import Keys



browser = webdriver.Firefox()



browser.get('http://www.yahoo.com')

assert 'Yahoo' in browser.title



elem = browser.find_element_by_name('p')  # Find the search box

elem.send_keys('seleniumhq' + Keys.RETURN)



browser.quit()

//do get all contensts

driver = webdriver.Chrome()

driver.get("http://www.ask.com")



print driver.title

inputElement = driver.find_element_by_name("q")



# type in the search

inputElement.send_keys("baseball hats")

# submit the form (although google automatically searches now without submitting)

inputElement.submit()



try:

    WebDriverWait(driver, 10).until(EC.title_contains("baseball"))

    print driver.title

    output = driver.page_source

    print(output)

finally:

    driver.quit()

 

iframes = driver.find_elements_by_tag_name("iframe")



for iframe in iframes

    driver.switch_to_default_content()

    driver.switch_to_frame(iframe)



    output = driver.page_source

    print(output)