From 626ff729e8dae925cd4d07a9261629527f8c93d3 Mon Sep 17 00:00:00 2001 From: xxchan Date: Mon, 3 Apr 2023 15:28:41 +0200 Subject: [PATCH] refactor(risedev): replace std::fs with fs_err (#8955) --- Cargo.lock | 1 + src/risedevtool/Cargo.toml | 1 + src/risedevtool/src/bin/risedev-compose.rs | 14 +++++++------- src/risedevtool/src/bin/risedev-config.rs | 2 +- src/risedevtool/src/bin/risedev-dev.rs | 8 ++++---- src/risedevtool/src/bin/risedev-docslt.rs | 12 ++++++------ src/risedevtool/src/compose.rs | 16 ++++++++-------- src/risedevtool/src/compose_deploy.rs | 13 ++++++------- src/risedevtool/src/config.rs | 2 +- src/risedevtool/src/config_gen/grafana_gen.rs | 2 +- src/risedevtool/src/task/etcd_service.rs | 2 +- src/risedevtool/src/task/grafana_service.rs | 16 ++++++++-------- src/risedevtool/src/task/kafka_service.rs | 6 +++--- src/risedevtool/src/task/minio_service.rs | 2 +- src/risedevtool/src/task/prometheus_service.rs | 2 +- src/risedevtool/src/task/zookeeper_service.rs | 6 +++--- src/risedevtool/src/wait.rs | 2 +- 17 files changed, 54 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd8c3a53e848f..ee0c49c7db59f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5662,6 +5662,7 @@ dependencies = [ "console", "dialoguer", "enum-iterator", + "fs-err", "google-cloud-pubsub", "indicatif", "isahc", diff --git a/src/risedevtool/Cargo.toml b/src/risedevtool/Cargo.toml index 6daaa80fa1087..89eba8b24fa62 100644 --- a/src/risedevtool/Cargo.toml +++ b/src/risedevtool/Cargo.toml @@ -20,6 +20,7 @@ clap = { version = "4", features = ["derive"] } console = "0.15" dialoguer = "0.10" enum-iterator = "1" +fs-err = "2.9.0" google-cloud-pubsub = "0.7.0" indicatif = "0.17" isahc = { version = "1", default-features = false, features = ["text-decoding"] } diff --git a/src/risedevtool/src/bin/risedev-compose.rs b/src/risedevtool/src/bin/risedev-compose.rs index fbce31d6bb376..e749df31ceaca 100644 --- a/src/risedevtool/src/bin/risedev-compose.rs +++ b/src/risedevtool/src/bin/risedev-compose.rs @@ -13,13 +13,13 @@ // limitations under the License. use std::collections::BTreeMap; -use std::fs::{self, File}; use std::io::Read; use std::path::Path; use anyhow::{anyhow, Result}; use clap::Parser; use console::style; +use fs_err::{self, File}; use itertools::Itertools; use risedev::{ compose_deploy, compute_risectl_env, Compose, ComposeConfig, ComposeDeployConfig, ComposeFile, @@ -92,7 +92,7 @@ fn main() -> Result<()> { let compose_config = ComposeConfig { image: load_docker_image_config( - &std::fs::read_to_string(RISEDEV_CONFIG_FILE)?, + &fs_err::read_to_string(RISEDEV_CONFIG_FILE)?, compose_deploy_config .as_ref() .and_then(|x| x.risingwave_image_override.as_ref()), @@ -164,7 +164,7 @@ fn main() -> Result<()> { .green(), style(&arg).green() )?; - fs::write( + fs_err::write( Path::new(&opts.directory).join("tpch-bench-args-frontend"), arg, )?; @@ -213,7 +213,7 @@ fn main() -> Result<()> { "-- Redpanda --\ntpch-bench: {}\n", style(&arg).green() )?; - fs::write( + fs_err::write( Path::new(&opts.directory).join("tpch-bench-args-kafka"), arg, )?; @@ -271,7 +271,7 @@ fn main() -> Result<()> { )?; } - fs::write( + fs_err::write( Path::new(&opts.directory).join(format!("{}.yml", node)), yaml, )?; @@ -290,7 +290,7 @@ fn main() -> Result<()> { println!("\n{}", log_buffer); - std::fs::write( + fs_err::write( Path::new(&opts.directory).join("_message.partial.sh"), log_buffer, )?; @@ -310,7 +310,7 @@ fn main() -> Result<()> { let yaml = serde_yaml::to_string(&compose_file)?; - fs::write(Path::new(&opts.directory).join("docker-compose.yml"), yaml)?; + fs_err::write(Path::new(&opts.directory).join("docker-compose.yml"), yaml)?; } Ok(()) diff --git a/src/risedevtool/src/bin/risedev-config.rs b/src/risedevtool/src/bin/risedev-config.rs index 29549541b6893..73864039f7196 100644 --- a/src/risedevtool/src/bin/risedev-config.rs +++ b/src/risedevtool/src/bin/risedev-config.rs @@ -14,7 +14,6 @@ #![allow(clippy::needless_question_mark)] -use std::fs::OpenOptions; use std::io::{BufRead, BufReader, BufWriter, Write}; use anyhow::{Context, Result}; @@ -22,6 +21,7 @@ use clap::{Parser, Subcommand, ValueEnum}; use console::style; use dialoguer::MultiSelect; use enum_iterator::{all, Sequence}; +use fs_err::OpenOptions; use itertools::Itertools; use risedev::RISEDEV_CONFIG_FILE; diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs index dfa8ea5d5be33..fbca7a63917bf 100644 --- a/src/risedevtool/src/bin/risedev-dev.rs +++ b/src/risedevtool/src/bin/risedev-dev.rs @@ -14,13 +14,13 @@ use std::env; use std::fmt::Write; -use std::fs::OpenOptions; use std::path::Path; use std::sync::Arc; use std::time::{Duration, Instant}; use anyhow::{anyhow, Context, Result}; use console::style; +use fs_err::OpenOptions; use indicatif::ProgressBar; use risedev::util::{complete_spin, fail_spin}; use risedev::{ @@ -376,14 +376,14 @@ fn main() -> Result<()> { if let Some(config_path) = &config_path { let target = Path::new(&env::var("PREFIX_CONFIG")?).join("risingwave.toml"); - std::fs::copy(config_path, target).context("config file not found")?; + fs_err::copy(config_path, target).context("config file not found")?; } { let mut out_str = String::new(); let mut emitter = YamlEmitter::new(&mut out_str); emitter.dump(&risedev_config)?; - std::fs::write( + fs_err::write( Path::new(&env::var("PREFIX_CONFIG")?).join("risedev-expanded.yml"), &out_str, )?; @@ -434,7 +434,7 @@ fn main() -> Result<()> { Err(_) => "".into(), }; - std::fs::write( + fs_err::write( Path::new(&env::var("PREFIX_CONFIG")?).join("risectl-env"), risectl_env, )?; diff --git a/src/risedevtool/src/bin/risedev-docslt.rs b/src/risedevtool/src/bin/risedev-docslt.rs index dcc7f6f273b83..a78121866b981 100644 --- a/src/risedevtool/src/bin/risedev-docslt.rs +++ b/src/risedevtool/src/bin/risedev-docslt.rs @@ -130,9 +130,9 @@ fn generate_slt_files(package_name: &str) -> Result<()> { } } - let rustdoc: JsonValue = serde_json::from_reader(std::io::BufReader::new( - std::fs::File::open(format!("target/doc/{}.json", package_name))?, - ))?; + let rustdoc: JsonValue = serde_json::from_reader(std::io::BufReader::new(fs_err::File::open( + format!("target/doc/{}.json", package_name), + )?))?; let index = rustdoc["index"] .as_object() .ok_or_else(|| anyhow!("failed to access `index` field as object"))?; @@ -174,9 +174,9 @@ fn generate_slt_files(package_name: &str) -> Result<()> { } let slt_dir = PathBuf::from(format!("e2e_test/generated/docslt/{}", package_name)); - std::fs::remove_dir_all(&slt_dir).ok(); + fs_err::remove_dir_all(&slt_dir).ok(); if !slt_blocks_per_file.is_empty() { - std::fs::create_dir_all(&slt_dir)?; + fs_err::create_dir_all(&slt_dir)?; } for filename in slt_blocks_per_file.keys() { @@ -186,7 +186,7 @@ fn generate_slt_files(package_name: &str) -> Result<()> { .components() .filter_map(|comp| comp.as_os_str().to_str().filter(|s| *s != "src")) .join("__"); - let mut slt_file = std::fs::File::create(slt_dir.join(slt_filename))?; + let mut slt_file = fs_err::File::create(slt_dir.join(slt_filename))?; write!( slt_file, "\ diff --git a/src/risedevtool/src/compose.rs b/src/risedevtool/src/compose.rs index afeb67d1b4b76..55a4a00f46b77 100644 --- a/src/risedevtool/src/compose.rs +++ b/src/risedevtool/src/compose.rs @@ -161,7 +161,7 @@ impl Compose for ComputeNodeConfig { if let Some(c) = &config.rw_config_path { let target = Path::new(&config.config_directory).join("risingwave.toml"); - std::fs::copy(c, target)?; + fs_err::copy(c, target)?; command.arg("--config-path").arg("/risingwave.toml"); } @@ -205,7 +205,7 @@ impl Compose for MetaNodeConfig { if let Some(c) = &config.rw_config_path { let target = Path::new(&config.config_directory).join("risingwave.toml"); - std::fs::copy(c, target)?; + fs_err::copy(c, target)?; command.arg("--config-path").arg("/risingwave.toml"); } @@ -238,7 +238,7 @@ impl Compose for FrontendConfig { if let Some(c) = &config.rw_config_path { let target = Path::new(&config.config_directory).join("risingwave.toml"); - std::fs::copy(c, target)?; + fs_err::copy(c, target)?; command.arg("--config-path").arg("/risingwave.toml"); } @@ -268,7 +268,7 @@ impl Compose for CompactorConfig { if let Some(c) = &config.rw_config_path { let target = Path::new(&config.config_directory).join("risingwave.toml"); - std::fs::copy(c, target)?; + fs_err::copy(c, target)?; command.arg("--config-path").arg("/risingwave.toml"); } @@ -404,7 +404,7 @@ impl Compose for PrometheusConfig { ..Default::default() }; - std::fs::write( + fs_err::write( Path::new(&config.config_directory).join("prometheus.yaml"), prometheus_config, )?; @@ -419,17 +419,17 @@ impl Compose for PrometheusConfig { impl Compose for GrafanaConfig { fn compose(&self, config: &ComposeConfig) -> Result { let config_root = Path::new(&config.config_directory); - std::fs::write( + fs_err::write( config_root.join("grafana.ini"), GrafanaGen.gen_custom_ini(self), )?; - std::fs::write( + fs_err::write( config_root.join("grafana-risedev-datasource.yml"), GrafanaGen.gen_datasource_yml(self)?, )?; - std::fs::write( + fs_err::write( config_root.join("grafana-risedev-dashboard.yml"), GrafanaGen.gen_dashboard_yml(self, config_root, "/")?, )?; diff --git a/src/risedevtool/src/compose_deploy.rs b/src/risedevtool/src/compose_deploy.rs index b89b538de55ac..9b5e670c2b8ad 100644 --- a/src/risedevtool/src/compose_deploy.rs +++ b/src/risedevtool/src/compose_deploy.rs @@ -13,7 +13,6 @@ // limitations under the License. use std::collections::{BTreeMap, HashMap}; -use std::fs; use std::os::unix::prelude::PermissionsExt; use std::path::Path; @@ -136,7 +135,7 @@ fi r#"echo "{id}: $(tput setaf 2)done sync config$(tput sgr0)""#, )?; let sh = format!("_deploy.{id}.partial.sh"); - std::fs::write(Path::new(output_directory).join(&sh), y)?; + fs_err::write(Path::new(output_directory).join(&sh), y)?; writeln!(x, "{sh}")?; } writeln!(x, "EOF")?; @@ -182,7 +181,7 @@ fi } let sh = format!("_stop.{id}.partial.sh"); - std::fs::write(Path::new(output_directory).join(&sh), y)?; + fs_err::write(Path::new(output_directory).join(&sh), y)?; writeln!(x, "{sh}")?; } writeln!(x, "EOF")?; @@ -233,7 +232,7 @@ fi writeln!(y, "ssh {ssh_extra_args} ubuntu@{public_ip} \"bash -c 'cd {base_folder} && docker compose ps'\"")?; let sh = format!("_check.{id}.partial.sh"); - std::fs::write(Path::new(output_directory).join(&sh), y)?; + fs_err::write(Path::new(output_directory).join(&sh), y)?; writeln!(x, "{sh}")?; } writeln!(x, "EOF")?; @@ -241,9 +240,9 @@ fi x }; let deploy_sh = Path::new(output_directory).join("deploy.sh"); - fs::write(&deploy_sh, shell_script)?; - let mut perms = fs::metadata(&deploy_sh)?.permissions(); + fs_err::write(&deploy_sh, shell_script)?; + let mut perms = fs_err::metadata(&deploy_sh)?.permissions(); perms.set_mode(perms.mode() | 0o755); - fs::set_permissions(&deploy_sh, perms)?; + fs_err::set_permissions(&deploy_sh, perms)?; Ok(()) } diff --git a/src/risedevtool/src/config.rs b/src/risedevtool/src/config.rs index cb29dc9bb804f..b85784834832b 100644 --- a/src/risedevtool/src/config.rs +++ b/src/risedevtool/src/config.rs @@ -41,7 +41,7 @@ impl ConfigExpander { /// Load a single document YAML file. fn load_yaml(path: impl AsRef) -> Result { let path = path.as_ref(); - let content = std::fs::read_to_string(path)?; + let content = fs_err::read_to_string(path)?; let [config]: [_; 1] = YamlLoader::load_from_str(&content)? .try_into() .map_err(|_| anyhow!("expect `{}` to have only one section", path.display()))?; diff --git a/src/risedevtool/src/config_gen/grafana_gen.rs b/src/risedevtool/src/config_gen/grafana_gen.rs index 7dfd740d5fbd2..eebd3a5673b52 100644 --- a/src/risedevtool/src/config_gen/grafana_gen.rs +++ b/src/risedevtool/src/config_gen/grafana_gen.rs @@ -90,7 +90,7 @@ datasources: let filename = "risingwave-dashboard.json"; let generate_path = generate_path.as_ref(); let dashboard_path = Path::new(generate_path).join(filename); - std::fs::copy("grafana/risingwave-dashboard.json", dashboard_path)?; + fs_err::copy("grafana/risingwave-dashboard.json", dashboard_path)?; let grafana_read_path = grafana_read_path.as_ref(); let dashboard_path = Path::new(grafana_read_path).join(filename); diff --git a/src/risedevtool/src/task/etcd_service.rs b/src/risedevtool/src/task/etcd_service.rs index 1912ce58f5b1b..e9066db27bd1b 100644 --- a/src/risedevtool/src/task/etcd_service.rs +++ b/src/risedevtool/src/task/etcd_service.rs @@ -109,7 +109,7 @@ impl Task for EtcdService { Self::apply_command_args(&mut cmd, &self.config)?; let path = Path::new(&env::var("PREFIX_DATA")?).join(self.id()); - std::fs::create_dir_all(&path)?; + fs_err::create_dir_all(&path)?; cmd.arg("--data-dir").arg(&path); ctx.run_command(ctx.tmux_run(cmd)?)?; diff --git a/src/risedevtool/src/task/grafana_service.rs b/src/risedevtool/src/task/grafana_service.rs index 4e212300cb3c6..57165f3e2d518 100644 --- a/src/risedevtool/src/task/grafana_service.rs +++ b/src/risedevtool/src/task/grafana_service.rs @@ -57,28 +57,28 @@ impl GrafanaService { ) -> Result<()> { let config_root = config_root.as_ref(); - std::fs::write( + fs_err::write( config_root.join("custom.ini"), GrafanaGen.gen_custom_ini(config), )?; let config_datasources_dir = config_root.join("provisioning").join("datasources"); - std::fs::remove_dir_all(&config_datasources_dir)?; - std::fs::create_dir_all(&config_datasources_dir)?; - std::fs::write( + fs_err::remove_dir_all(&config_datasources_dir)?; + fs_err::create_dir_all(&config_datasources_dir)?; + fs_err::write( config_datasources_dir.join("risedev-prometheus.yml"), GrafanaGen.gen_datasource_yml(config)?, )?; let prefix_config = prefix_config.as_ref(); let config_dashboards_dir = config_root.join("provisioning").join("dashboards"); - std::fs::remove_dir_all(&config_dashboards_dir)?; - std::fs::create_dir_all(&config_dashboards_dir)?; - std::fs::write( + fs_err::remove_dir_all(&config_dashboards_dir)?; + fs_err::create_dir_all(&config_dashboards_dir)?; + fs_err::write( config_dashboards_dir.join("risingwave-dashboard.yaml"), GrafanaGen.gen_dashboard_yml(config, prefix_config, prefix_config)?, )?; - // std::fs::write( + // fs_err::write( // config_dashboards_dir.join("aws-s3-dashboards.yaml"), // &GrafanaGen.gen_s3_dashboard_yml(config, prefix_config)?, // )?; diff --git a/src/risedevtool/src/task/kafka_service.rs b/src/risedevtool/src/task/kafka_service.rs index 62cb30916bd74..ceb96a8371645 100644 --- a/src/risedevtool/src/task/kafka_service.rs +++ b/src/risedevtool/src/task/kafka_service.rs @@ -59,13 +59,13 @@ impl Task for KafkaService { Path::new(&env::var("PREFIX_DATA")?).join(self.id()) } else { let path = Path::new("/tmp/risedev").join(self.id()); - std::fs::remove_dir_all(&path).ok(); + fs_err::remove_dir_all(&path).ok(); path }; - std::fs::create_dir_all(&path)?; + fs_err::create_dir_all(&path)?; let config_path = Path::new(&prefix_config).join(format!("{}.properties", self.id())); - std::fs::write( + fs_err::write( &config_path, KafkaGen.gen_server_properties(&self.config, &path.to_string_lossy()), )?; diff --git a/src/risedevtool/src/task/minio_service.rs b/src/risedevtool/src/task/minio_service.rs index f3d8c31a83774..e6090047c4ee6 100644 --- a/src/risedevtool/src/task/minio_service.rs +++ b/src/risedevtool/src/task/minio_service.rs @@ -90,7 +90,7 @@ impl Task for MinioService { let prefix_config = env::var("PREFIX_CONFIG")?; let data_path = Path::new(&env::var("PREFIX_DATA")?).join(self.id()); - std::fs::create_dir_all(&data_path)?; + fs_err::create_dir_all(&data_path)?; cmd.arg("--config-dir") .arg(Path::new(&prefix_config).join("minio")) diff --git a/src/risedevtool/src/task/prometheus_service.rs b/src/risedevtool/src/task/prometheus_service.rs index 453cc137df74b..26907f5e94c56 100644 --- a/src/risedevtool/src/task/prometheus_service.rs +++ b/src/risedevtool/src/task/prometheus_service.rs @@ -63,7 +63,7 @@ impl Task for PrometheusService { let prefix_config = env::var("PREFIX_CONFIG")?; let prefix_data = env::var("PREFIX_DATA")?; - std::fs::write( + fs_err::write( Path::new(&prefix_config).join("prometheus.yml"), PrometheusGen.gen_prometheus_yml(&self.config), )?; diff --git a/src/risedevtool/src/task/zookeeper_service.rs b/src/risedevtool/src/task/zookeeper_service.rs index 5d5138186c9fb..14ce022adaeea 100644 --- a/src/risedevtool/src/task/zookeeper_service.rs +++ b/src/risedevtool/src/task/zookeeper_service.rs @@ -59,13 +59,13 @@ impl Task for ZooKeeperService { Path::new(&env::var("PREFIX_DATA")?).join(self.id()) } else { let path = Path::new("/tmp/risedev").join(self.id()); - std::fs::remove_dir_all(&path).ok(); + fs_err::remove_dir_all(&path).ok(); path }; - std::fs::create_dir_all(&path)?; + fs_err::create_dir_all(&path)?; let config_path = Path::new(&prefix_config).join(format!("{}.properties", self.id())); - std::fs::write( + fs_err::write( &config_path, ZooKeeperGen.gen_server_properties(&self.config, &path.to_string_lossy()), )?; diff --git a/src/risedevtool/src/wait.rs b/src/risedevtool/src/wait.rs index f29d42b2ad7c9..3a47c7a4b6432 100644 --- a/src/risedevtool/src/wait.rs +++ b/src/risedevtool/src/wait.rs @@ -54,7 +54,7 @@ pub fn wait( if detect_failure && p.exists() { let mut buf = String::new(); - std::fs::File::open(p)?.read_to_string(&mut buf)?; + fs_err::File::open(p)?.read_to_string(&mut buf)?; return Err(anyhow!( "{} exited while waiting for connection: {}",