From eafe5dfdc02292bf284c78f17a2fb2253d926910 Mon Sep 17 00:00:00 2001 From: Jack McCluskey <34928439+jrmccluskey@users.noreply.github.com> Date: Thu, 20 Jul 2023 13:51:43 -0400 Subject: [PATCH] Reduce Python INFO logging verbosity at start-up (#27577) * Reduce Python INFO logging verbosity at start-up * Formatting --- .../apache_beam/runners/worker/sdk_worker_main.py | 4 ++-- sdks/python/container/boot.go | 10 +++++----- sdks/python/container/piputil.go | 11 ++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py index abbcfb72382b..87cf06e862ab 100644 --- a/sdks/python/apache_beam/runners/worker/sdk_worker_main.py +++ b/sdks/python/apache_beam/runners/worker/sdk_worker_main.py @@ -52,7 +52,7 @@ def _import_beam_plugins(plugins): for plugin in plugins: try: importlib.import_module(plugin) - _LOGGER.info('Imported beam-plugin %s', plugin) + _LOGGER.debug('Imported beam-plugin %s', plugin) except ImportError: try: _LOGGER.debug(( @@ -61,7 +61,7 @@ def _import_beam_plugins(plugins): plugin) module, _ = plugin.rsplit('.', 1) importlib.import_module(module) - _LOGGER.info('Imported %s for beam-plugin %s', module, plugin) + _LOGGER.debug('Imported %s for beam-plugin %s', module, plugin) except ImportError as exc: _LOGGER.warning('Failed to import beam-plugin %s', plugin, exc_info=exc) diff --git a/sdks/python/container/boot.go b/sdks/python/container/boot.go index 1e70e0db1513..da2f3cc28f54 100644 --- a/sdks/python/container/boot.go +++ b/sdks/python/container/boot.go @@ -164,11 +164,11 @@ func launchSDKProcess() error { if err != nil { return errors.New( "failed to create a virtual environment. If running on Ubuntu systems, " + - "you might need to install `python3-venv` package. " + - "To run the SDK process in default environment instead, " + - "set the environment variable `RUN_PYTHON_SDK_IN_DEFAULT_ENVIRONMENT=1`. " + - "In custom Docker images, you can do that with an `ENV` statement. " + - fmt.Sprintf("Encountered error: %v", err)) + "you might need to install `python3-venv` package. " + + "To run the SDK process in default environment instead, " + + "set the environment variable `RUN_PYTHON_SDK_IN_DEFAULT_ENVIRONMENT=1`. " + + "In custom Docker images, you can do that with an `ENV` statement. " + + fmt.Sprintf("Encountered error: %v", err)) } cleanupFunc := func() { os.RemoveAll(venvDir) diff --git a/sdks/python/container/piputil.go b/sdks/python/container/piputil.go index a00e017445e3..c9e396b0e6a4 100644 --- a/sdks/python/container/piputil.go +++ b/sdks/python/container/piputil.go @@ -37,14 +37,15 @@ func pipInstallRequirements(files []string, dir, name string) error { // as possible PyPI downloads. In the first round the --find-links // option will make sure that only things staged in the worker will be // used without following their dependencies. - args := []string{"-m", "pip", "install", "-r", filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check", "--no-index", "--no-deps", "--find-links", dir} + args := []string{"-m", "pip", "install", "-r", filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check", "--no-index", "--no-deps", "--find-links", + "-q", dir} if err := execx.Execute("python", args...); err != nil { fmt.Println("Some packages could not be installed solely from the requirements cache. Installing packages from PyPI.") } // The second install round opens up the search for packages on PyPI and // also installs dependencies. The key is that if all the packages have // been installed in the first round then this command will be a no-op. - args = []string{"-m", "pip", "install", "-r", filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check", "--find-links", dir} + args = []string{"-m", "pip", "install", "-r", filepath.Join(dir, name), "--no-cache-dir", "--disable-pip-version-check", "--find-links", "-q", dir} return execx.Execute("python", args...) } } @@ -76,18 +77,18 @@ func pipInstallPackage(files []string, dir, name string, force, optional bool, e // installed version will match the package specified, the package itself // will not be reinstalled, but its dependencies will now be resolved and // installed if necessary. This achieves our goal outlined above. - args := []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", "--upgrade", "--force-reinstall", "--no-deps", + args := []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", "--upgrade", "--force-reinstall", "--no-deps", "-q", filepath.Join(dir, packageSpec)} err := execx.Execute("python", args...) if err != nil { return err } - args = []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", filepath.Join(dir, packageSpec)} + args = []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", "-q", filepath.Join(dir, packageSpec)} return execx.Execute("python", args...) } // Case when we do not perform a forced reinstall. - args := []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", filepath.Join(dir, packageSpec)} + args := []string{"-m", "pip", "install", "--no-cache-dir", "--disable-pip-version-check", "-q", filepath.Join(dir, packageSpec)} return execx.Execute("python", args...) } }