webdriver.py 9.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257
  1. from __future__ import annotations
  2. try:
  3. from platformdirs import user_config_dir
  4. from undetected_chromedriver import Chrome, ChromeOptions, find_chrome_executable
  5. from selenium.webdriver.remote.webdriver import WebDriver
  6. from selenium.webdriver.remote.webelement import WebElement
  7. from selenium.webdriver.common.by import By
  8. from selenium.webdriver.support.ui import WebDriverWait
  9. from selenium.webdriver.support import expected_conditions as EC
  10. from selenium.webdriver.common.keys import Keys
  11. from selenium.common.exceptions import NoSuchElementException
  12. has_requirements = True
  13. except ImportError:
  14. from typing import Type as WebDriver
  15. has_requirements = False
  16. import time
  17. from shutil import which
  18. from os import path
  19. from os import access, R_OK
  20. from .typing import Cookies
  21. from .errors import MissingRequirementsError
  22. from . import debug
  23. try:
  24. from pyvirtualdisplay import Display
  25. has_pyvirtualdisplay = True
  26. except ImportError:
  27. has_pyvirtualdisplay = False
  28. try:
  29. from undetected_chromedriver import Chrome as _Chrome, ChromeOptions
  30. from seleniumwire.webdriver import InspectRequestsMixin, DriverCommonMixin
  31. class Chrome(InspectRequestsMixin, DriverCommonMixin, _Chrome):
  32. def __init__(self, *args, options=None, seleniumwire_options={}, **kwargs):
  33. if options is None:
  34. options = ChromeOptions()
  35. config = self._setup_backend(seleniumwire_options)
  36. options.add_argument(f"--proxy-server={config['proxy']['httpProxy']}")
  37. options.add_argument("--proxy-bypass-list=<-loopback>")
  38. options.add_argument("--ignore-certificate-errors")
  39. super().__init__(*args, options=options, **kwargs)
  40. has_seleniumwire = True
  41. except:
  42. has_seleniumwire = False
  43. def get_browser(
  44. user_data_dir: str = None,
  45. headless: bool = False,
  46. proxy: str = None,
  47. options: ChromeOptions = None
  48. ) -> WebDriver:
  49. """
  50. Creates and returns a Chrome WebDriver with specified options.
  51. Args:
  52. user_data_dir (str, optional): Directory for user data. If None, uses default directory.
  53. headless (bool, optional): Whether to run the browser in headless mode. Defaults to False.
  54. proxy (str, optional): Proxy settings for the browser. Defaults to None.
  55. options (ChromeOptions, optional): ChromeOptions object with specific browser options. Defaults to None.
  56. Returns:
  57. WebDriver: An instance of WebDriver configured with the specified options.
  58. """
  59. if not has_requirements:
  60. raise MissingRequirementsError('Install Webdriver packages | pip install -U g4f[webdriver]')
  61. browser = find_chrome_executable()
  62. if browser is None:
  63. raise MissingRequirementsError('Install "Google Chrome" browser')
  64. if user_data_dir is None:
  65. user_data_dir = user_config_dir("g4f")
  66. if user_data_dir and debug.logging:
  67. print("Open browser with config dir:", user_data_dir)
  68. if not options:
  69. options = ChromeOptions()
  70. if proxy:
  71. options.add_argument(f'--proxy-server={proxy}')
  72. # Check for system driver in docker
  73. driver = which('chromedriver') or '/usr/bin/chromedriver'
  74. if not path.isfile(driver) or not access(driver, R_OK):
  75. driver = None
  76. return Chrome(
  77. options=options,
  78. user_data_dir=user_data_dir,
  79. driver_executable_path=driver,
  80. browser_executable_path=browser,
  81. headless=headless,
  82. patcher_force_close=True
  83. )
  84. def get_driver_cookies(driver: WebDriver) -> Cookies:
  85. """
  86. Retrieves cookies from the specified WebDriver.
  87. Args:
  88. driver (WebDriver): The WebDriver instance from which to retrieve cookies.
  89. Returns:
  90. dict: A dictionary containing cookies with their names as keys and values as cookie values.
  91. """
  92. return {cookie["name"]: cookie["value"] for cookie in driver.get_cookies()}
  93. def bypass_cloudflare(driver: WebDriver, url: str, timeout: int) -> None:
  94. """
  95. Attempts to bypass Cloudflare protection when accessing a URL using the provided WebDriver.
  96. Args:
  97. driver (WebDriver): The WebDriver to use for accessing the URL.
  98. url (str): The URL to access.
  99. timeout (int): Time in seconds to wait for the page to load.
  100. Raises:
  101. Exception: If there is an error while bypassing Cloudflare or loading the page.
  102. """
  103. driver.get(url)
  104. if driver.find_element(By.TAG_NAME, "body").get_attribute("class") == "no-js":
  105. if debug.logging:
  106. print("Cloudflare protection detected:", url)
  107. # Open website in a new tab
  108. element = driver.find_element(By.ID, "challenge-body-text")
  109. driver.execute_script(f"""
  110. arguments[0].addEventListener('click', () => {{
  111. window.open(arguments[1]);
  112. }});
  113. """, element, url)
  114. element.click()
  115. time.sleep(5)
  116. # Switch to the new tab and close the old tab
  117. original_window = driver.current_window_handle
  118. for window_handle in driver.window_handles:
  119. if window_handle != original_window:
  120. driver.close()
  121. driver.switch_to.window(window_handle)
  122. break
  123. # Click on the challenge button in the iframe
  124. try:
  125. driver.switch_to.frame(driver.find_element(By.CSS_SELECTOR, "#turnstile-wrapper iframe"))
  126. WebDriverWait(driver, 5).until(
  127. EC.presence_of_element_located((By.CSS_SELECTOR, "#challenge-stage input"))
  128. ).click()
  129. except NoSuchElementException:
  130. ...
  131. except Exception as e:
  132. if debug.logging:
  133. print(f"Error bypassing Cloudflare: {str(e).splitlines()[0]}")
  134. #driver.switch_to.default_content()
  135. driver.switch_to.window(window_handle)
  136. driver.execute_script("document.href = document.href;")
  137. WebDriverWait(driver, timeout).until(
  138. EC.presence_of_element_located((By.CSS_SELECTOR, "body:not(.no-js)"))
  139. )
  140. class WebDriverSession:
  141. """
  142. Manages a Selenium WebDriver session, including handling of virtual displays and proxies.
  143. """
  144. def __init__(
  145. self,
  146. webdriver: WebDriver = None,
  147. user_data_dir: str = None,
  148. headless: bool = False,
  149. virtual_display: bool = False,
  150. proxy: str = None,
  151. options: ChromeOptions = None
  152. ):
  153. """
  154. Initializes a new instance of the WebDriverSession.
  155. Args:
  156. webdriver (WebDriver, optional): A WebDriver instance for the session. Defaults to None.
  157. user_data_dir (str, optional): Directory for user data. Defaults to None.
  158. headless (bool, optional): Whether to run the browser in headless mode. Defaults to False.
  159. virtual_display (bool, optional): Whether to use a virtual display. Defaults to False.
  160. proxy (str, optional): Proxy settings for the browser. Defaults to None.
  161. options (ChromeOptions, optional): ChromeOptions for the browser. Defaults to None.
  162. """
  163. self.webdriver = webdriver
  164. self.user_data_dir = user_data_dir
  165. self.headless = headless
  166. self.virtual_display = Display(size=(1920, 1080)) if has_pyvirtualdisplay and virtual_display else None
  167. self.proxy = proxy
  168. self.options = options
  169. self.default_driver = None
  170. def reopen(
  171. self,
  172. user_data_dir: str = None,
  173. headless: bool = False,
  174. virtual_display: bool = False
  175. ) -> WebDriver:
  176. """
  177. Reopens the WebDriver session with new settings.
  178. Args:
  179. user_data_dir (str, optional): Directory for user data. Defaults to current value.
  180. headless (bool, optional): Whether to run the browser in headless mode. Defaults to current value.
  181. virtual_display (bool, optional): Whether to use a virtual display. Defaults to current value.
  182. Returns:
  183. WebDriver: The reopened WebDriver instance.
  184. """
  185. user_data_dir = user_data_dir or self.user_data_dir
  186. if self.default_driver:
  187. self.default_driver.quit()
  188. if not virtual_display and self.virtual_display:
  189. self.virtual_display.stop()
  190. self.virtual_display = None
  191. self.default_driver = get_browser(user_data_dir, headless, self.proxy)
  192. return self.default_driver
  193. def __enter__(self) -> WebDriver:
  194. """
  195. Context management method for entering a session. Initializes and returns a WebDriver instance.
  196. Returns:
  197. WebDriver: An instance of WebDriver for this session.
  198. """
  199. if self.webdriver:
  200. return self.webdriver
  201. if self.virtual_display:
  202. self.virtual_display.start()
  203. self.default_driver = get_browser(self.user_data_dir, self.headless, self.proxy, self.options)
  204. return self.default_driver
  205. def __exit__(self, exc_type, exc_val, exc_tb):
  206. """
  207. Context management method for exiting a session. Closes and quits the WebDriver.
  208. Args:
  209. exc_type: Exception type.
  210. exc_val: Exception value.
  211. exc_tb: Exception traceback.
  212. Note:
  213. Closes the WebDriver and stops the virtual display if used.
  214. """
  215. if self.default_driver:
  216. try:
  217. self.default_driver.close()
  218. except Exception as e:
  219. if debug.logging:
  220. print(f"Error closing WebDriver: {str(e).splitlines()[0]}")
  221. finally:
  222. self.default_driver.quit()
  223. if self.virtual_display:
  224. self.virtual_display.stop()
  225. def element_send_text(element: WebElement, text: str) -> None:
  226. script = "arguments[0].innerText = arguments[1];"
  227. element.parent.execute_script(script, element, text)
  228. element.send_keys(Keys.ENTER)