[rust] Use CfT endpoints to discover chromedriver 115+ (#12208)

rust/src/chrome.rs

@@ -17,35 +17,41 @@
 
 use crate::config::ManagerConfig;
 use reqwest::Client;
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::error::Error;
+use std::option::Option;
 use std::path::PathBuf;
 
-use crate::config::ARCH::ARM64;
+use crate::config::ARCH::{ARM64, X32};
 use crate::config::OS::{LINUX, MACOS, WINDOWS};
-use crate::downloads::read_version_from_link;
-use crate::files::{compose_driver_path_in_cache, BrowserPath, PARSE_ERROR};
+use crate::downloads::{parse_json_from_url, read_version_from_link};
+use crate::files::{compose_driver_path_in_cache, BrowserPath};
 use crate::logger::Logger;
 use crate::metadata::{
     create_driver_metadata, get_driver_version_from_metadata, get_metadata, write_metadata,
 };
 use crate::{
     create_http_client, format_one_arg, format_three_args, SeleniumManager, BETA,
-    DASH_DASH_VERSION, DEV, ENV_LOCALAPPDATA, ENV_PROGRAM_FILES, ENV_PROGRAM_FILES_X86,
-    FALLBACK_RETRIES, NIGHTLY, REG_QUERY, REMOVE_X86, STABLE, WMIC_COMMAND, WMIC_COMMAND_ENV,
+    DASH_DASH_VERSION, DEV, ENV_LOCALAPPDATA, ENV_PROGRAM_FILES, ENV_PROGRAM_FILES_X86, NIGHTLY,
+    REG_QUERY, REMOVE_X86, STABLE, WMIC_COMMAND, WMIC_COMMAND_ENV,
 };
 
 pub const CHROME_NAME: &str = "chrome";
 pub const CHROMEDRIVER_NAME: &str = "chromedriver";
 const DRIVER_URL: &str = "https://chromedriver.storage.googleapis.com/";
 const LATEST_RELEASE: &str = "LATEST_RELEASE";
+const CFT_URL: &str = "https://googlechromelabs.github.io/chrome-for-testing/";
+const GOOD_VERSIONS_ENDPOINT: &str = "known-good-versions-with-downloads.json";
+const LATEST_VERSIONS_ENDPOINT: &str = "last-known-good-versions-with-downloads.json";
 
 pub struct ChromeManager {
     pub browser_name: &'static str,
     pub driver_name: &'static str,
     pub config: ManagerConfig,
     pub http_client: Client,
     pub log: Logger,
+    pub driver_url: Option<String>,
 }
 
 impl ChromeManager {
@@ -61,8 +67,143 @@ impl ChromeManager {
             http_client: create_http_client(default_timeout, default_proxy)?,
             config,
             log: Logger::default(),
+            driver_url: None,
         }))
     }
+
+    fn create_latest_release_url(&self) -> String {
+        format!("{}{}", DRIVER_URL, LATEST_RELEASE)
+    }
+
+    fn create_latest_release_with_version_url(&self) -> String {
+        format!(
+            "{}{}_{}",
+            DRIVER_URL,
+            LATEST_RELEASE,
+            self.get_major_browser_version()
+        )
+    }
+
+    fn create_good_versions_url(&self) -> String {
+        format!("{}{}", CFT_URL, GOOD_VERSIONS_ENDPOINT)
+    }
+
+    fn create_latest_versions_url(&self) -> String {
+        format!("{}{}", CFT_URL, LATEST_VERSIONS_ENDPOINT)
+    }
+
+    fn request_driver_version_from_latest(
+        &self,
+        driver_url: String,
+    ) -> Result<String, Box<dyn Error>> {
+        self.log.debug(format!(
+            "Reading {} version from {}",
+            &self.driver_name, driver_url
+        ));
+        read_version_from_link(self.get_http_client(), driver_url, self.get_logger())
+    }
+
+    fn request_versions_from_cft<T>(&self, driver_url: String) -> Result<T, Box<dyn Error>>
+    where
+        T: Serialize + for<'a> Deserialize<'a>,
+    {
+        self.log.debug(format!(
+            "Reading {} metadata from {}",
+            &self.driver_name, driver_url
+        ));
+        parse_json_from_url::<T>(self.get_http_client(), driver_url)
+    }
+
+    fn request_latest_driver_version_from_cft(&mut self) -> Result<String, Box<dyn Error>> {
+        let versions_with_downloads = self
+            .request_versions_from_cft::<LatestVersionsWithDownloads>(
+                self.create_latest_versions_url(),
+            )?;
+
+        let stable_channel = versions_with_downloads.channels.stable;
+        let chromedriver = stable_channel.downloads.chromedriver;
+        if chromedriver.is_none() {
+            // This should be temporal, since currently the stable channel has no chromedriver download
+            self.log.warn(format!(
+                "Latest stable version of {} not found using CfT endpoints. Trying with {}",
+                &self.driver_name, LATEST_RELEASE
+            ));
+            return self.request_driver_version_from_latest(self.create_latest_release_url());
+        }
+
+        let url: Vec<&PlatformUrl> = chromedriver
+            .as_ref()
+            .unwrap()
+            .iter()
+            .filter(|p| p.platform.eq_ignore_ascii_case(self.get_platform_label()))
+            .collect();
+        self.log.trace(format!("URLs for CfT: {:?}", url));
+        self.driver_url = Some(url.first().unwrap().url.to_string());
+
+        Ok(stable_channel.version)
+    }
+
+    fn request_good_version_from_cft(&mut self) -> Result<String, Box<dyn Error>> {
+        let browser_or_driver_version = if self.get_driver_version().is_empty() {
+            self.get_browser_version()
+        } else {
+            self.get_driver_version()
+        };
+        let version_for_filtering = self.get_major_version(browser_or_driver_version)?;
+        self.log.trace(format!(
+            "Driver version used to request CfT: {version_for_filtering}"
+        ));
+
+        let all_versions = self
+            .request_versions_from_cft::<VersionsWithDownloads>(self.create_good_versions_url())?;
+        let filtered_versions: Vec<Version> = all_versions
+            .versions
+            .into_iter()
+            .filter(|r| r.version.starts_with(version_for_filtering.as_str()))
+            .collect();
+        if filtered_versions.is_empty() {
+            return Err(format!(
+                "{} {} not available",
+                self.get_driver_name(),
+                version_for_filtering
+            )
+            .into());
+        }
+
+        let driver_version = filtered_versions.last().unwrap();
+        let url: Vec<&PlatformUrl> = driver_version
+            .downloads
+            .chromedriver
+            .as_ref()
+            .unwrap()
+            .iter()
+            .filter(|p| p.platform.eq_ignore_ascii_case(self.get_platform_label()))
+            .collect();
+        self.log.trace(format!("URLs for CfT: {:?}", url));
+        self.driver_url = Some(url.first().unwrap().url.to_string());
+
+        Ok(driver_version.version.to_string())
+    }
+
+    fn get_platform_label(&self) -> &str {
+        let os = self.get_os();
+        let arch = self.get_arch();
+        if WINDOWS.is(os) {
+            if X32.is(arch) {
+                "win32"
+            } else {
+                "win64"
+            }
+        } else if MACOS.is(os) {
+            if ARM64.is(arch) {
+                "mac-arm64"
+            } else {
+                "mac-x64"
+            }
+        } else {
+            "linux64"
+        }
+    }
 }
 
 impl SeleniumManager for ChromeManager {
@@ -162,8 +303,9 @@ impl SeleniumManager for ChromeManager {
         self.driver_name
     }
 
-    fn request_driver_version(&self) -> Result<String, Box<dyn Error>> {
-        let browser_version = self.get_browser_version();
+    fn request_driver_version(&mut self) -> Result<String, Box<dyn Error>> {
+        let browser_version_binding = self.get_major_browser_version();
+        let browser_version = browser_version_binding.as_str();
         let mut metadata = get_metadata(self.get_logger());
         let driver_ttl = self.get_config().driver_ttl;
 
@@ -177,46 +319,23 @@ impl SeleniumManager for ChromeManager {
                 Ok(driver_version)
             }
             _ => {
-                let mut driver_version = "".to_string();
-                let mut browser_version_int = browser_version.parse::<i32>().unwrap_or_default();
-                for i in 0..FALLBACK_RETRIES {
-                    let driver_url = if browser_version.is_empty() {
-                        format!("{}{}", DRIVER_URL, LATEST_RELEASE)
-                    } else {
-                        format!("{}{}_{}", DRIVER_URL, LATEST_RELEASE, browser_version_int)
-                    };
-                    if !browser_version.is_empty() && browser_version_int <= 0 {
-                        break;
-                    }
-                    self.log.debug(format!(
-                        "Reading {} version from {}",
-                        &self.driver_name, driver_url
-                    ));
-                    match read_version_from_link(
-                        self.get_http_client(),
-                        driver_url,
-                        self.get_logger(),
-                    ) {
-                        Ok(version) => {
-                            driver_version = version;
-                            break;
-                        }
-                        Err(err) => {
-                            if !err.to_string().eq(PARSE_ERROR) {
-                                return Err(err);
-                            }
-                            self.log.warn(format!(
-                                "Error getting version of {} {}. Retrying with {} {} (attempt {}/{})",
-                                &self.driver_name,
-                                browser_version_int,
-                                &self.driver_name,
-                                browser_version_int - 1,
-                                i + 1, FALLBACK_RETRIES
-                            ));
-                            browser_version_int -= 1;
-                        }
-                    }
-                }
+                let major_browser_version = browser_version.parse::<i32>().unwrap_or_default();
+                let driver_version = if !browser_version.is_empty() && major_browser_version < 115 {
+                    // For old versions (chromedriver 114-), the traditional method should work:
+                    // https://chromedriver.chromium.org/downloads
+                    self.request_driver_version_from_latest(
+                        self.create_latest_release_with_version_url(),
+                    )?
+                } else if browser_version.is_empty() {
+                    // For discovering the latest driver version, the CfT endpoints are also used
+                    self.request_latest_driver_version_from_cft()?
+                } else {
+                    // As of chromedriver 115+, the metadata for version discovery are published
+                    // by the "Chrome for Testing" (CfT) JSON endpoints:
+                    // https://googlechromelabs.github.io/chrome-for-testing/
+                    self.request_good_version_from_cft()?
+                };
+
                 if !browser_version.is_empty() && !driver_version.is_empty() {
                     metadata.drivers.push(create_driver_metadata(
                         browser_version,
@@ -231,7 +350,22 @@ impl SeleniumManager for ChromeManager {
         }
     }
 
-    fn get_driver_url(&self) -> Result<String, Box<dyn Error>> {
+    fn get_driver_url(&mut self) -> Result<String, Box<dyn Error>> {
+        let major_driver_version = self
+            .get_major_driver_version()
+            .parse::<i32>()
+            .unwrap_or_default();
+
+        if major_driver_version >= 115 && self.driver_url.is_none() {
+            // This case happens when driver_version is set (e.g. using CLI flag)
+            self.request_good_version_from_cft()?;
+        }
+
+        // As of Chrome 115+, the driver URL is already gathered thanks to the CfT endpoints
+        if self.driver_url.is_some() {
+            return Ok(self.driver_url.as_ref().unwrap().to_string());
+        }
+
         let driver_version = self.get_driver_version();
         let os = self.get_os();
         let arch = self.get_arch();
@@ -241,10 +375,6 @@ impl SeleniumManager for ChromeManager {
             if ARM64.is(arch) {
                 // As of chromedriver 106, the naming convention for macOS ARM64 releases changed. See:
                 // https://groups.google.com/g/chromedriver-users/c/JRuQzH3qr2c
-                let major_driver_version = self
-                    .get_major_version(driver_version)?
-                    .parse::<i32>()
-                    .unwrap_or_default();
                 if major_driver_version < 106 {
                     "mac64_m1"
                 } else {
@@ -265,18 +395,7 @@ impl SeleniumManager for ChromeManager {
     fn get_driver_path_in_cache(&self) -> PathBuf {
         let driver_version = self.get_driver_version();
         let os = self.get_os();
-        let arch = self.get_arch();
-        let arch_folder = if WINDOWS.is(os) {
-            "win32"
-        } else if MACOS.is(os) {
-            if ARM64.is(arch) {
-                "mac-arm64"
-            } else {
-                "mac64"
-            }
-        } else {
-            "linux64"
-        };
+        let arch_folder = self.get_platform_label();
         compose_driver_path_in_cache(self.driver_name, os, arch_folder, driver_version)
     }
 
@@ -300,3 +419,54 @@ impl SeleniumManager for ChromeManager {
         self.log = log;
     }
 }
+
+#[derive(Serialize, Deserialize)]
+pub struct LatestVersionsWithDownloads {
+    pub timestamp: String,
+    pub channels: Channels,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct Channels {
+    #[serde(rename = "Stable")]
+    pub stable: Channel,
+    #[serde(rename = "Beta")]
+    pub beta: Channel,
+    #[serde(rename = "Dev")]
+    pub dev: Channel,
+    #[serde(rename = "Canary")]
+    pub canary: Channel,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct Channel {
+    pub channel: String,
+    pub version: String,
+    pub revision: String,
+    pub downloads: Downloads,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct VersionsWithDownloads {
+    pub timestamp: String,
+    pub versions: Vec<Version>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct Version {
+    pub version: String,
+    pub revision: String,
+    pub downloads: Downloads,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct Downloads {
+    pub chrome: Vec<PlatformUrl>,
+    pub chromedriver: Option<Vec<PlatformUrl>>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct PlatformUrl {
+    pub platform: String,
+    pub url: String,
+}

rust/src/downloads.rs

@@ -16,6 +16,7 @@
 // under the License.
 
 use reqwest::Client;
+use serde::{Deserialize, Serialize};
 use std::error::Error;
 use std::fs::File;
 use std::io::copy;
@@ -92,3 +93,12 @@ pub async fn read_redirect_from_link(
         log,
     )
 }
+
+pub fn parse_json_from_url<T>(http_client: &Client, url: String) -> Result<T, Box<dyn Error>>
+where
+    T: Serialize + for<'a> Deserialize<'a>,
+{
+    let content = read_content_from_link(http_client, url)?;
+    let response: T = serde_json::from_str(&content)?;
+    Ok(response)
+}