From 32d8d060a660b0ec5702403da1b970118f62a314 Mon Sep 17 00:00:00 2001 From: Andrew Hlynskyi Date: Tue, 26 Nov 2024 21:56:04 +0200 Subject: [PATCH] feat: Radically redesign the build plan form (#646) --- package.py | 280 ++++++++++++++++++++----------------- tests/test_package_toml.py | 2 +- tests/test_zip_source.py | 12 +- 3 files changed, 159 insertions(+), 135 deletions(-) diff --git a/package.py b/package.py index 8cbeba30..09bc23e6 100644 --- a/package.py +++ b/package.py @@ -679,9 +679,10 @@ def plan(self, source_path, query): source_paths = [] build_plan = [] + build_step = [] def step(*x): - build_plan.append(x) + build_step.append(x) def hash(path): source_paths.append(path) @@ -754,6 +755,8 @@ def commands_step(path, commands): if path: path = os.path.normpath(path) + step("set:workdir", path) + batch = [] for c in commands: if isinstance(c, str): @@ -761,23 +764,21 @@ def commands_step(path, commands): if path: hash(path) if batch: - step("sh", path, "\n".join(batch)) + step("sh", "\n".join(batch)) batch.clear() c = shlex.split(c) - if len(c) == 3: + n = len(c) + if n == 3: _, _path, prefix = c prefix = prefix.strip() - _path = os.path.normpath(os.path.join(path, _path)) + _path = os.path.normpath(_path) step("zip:embedded", _path, prefix) - elif len(c) == 2: + elif n == 2: _, _path = c - prefix = None _path = os.path.normpath(_path) - step("zip:embedded", _path, prefix) - elif len(c) == 1: - prefix = None - _path = None - step("zip:embedded", _path, prefix) + step("zip:embedded", _path) + elif n == 1: + step("zip:embedded") else: raise ValueError( ":zip invalid call signature, use: " @@ -786,11 +787,9 @@ def commands_step(path, commands): else: batch.append(c) if batch: - step("sh", path, "\n".join(batch)) + step("sh", "\n".join(batch)) batch.clear() - step("reset:workdir") - for claim in claims: if isinstance(claim, str): path = claim @@ -877,12 +876,13 @@ def commands_step(path, commands): hash(path_from_pattern) else: hash(path) - - if patterns: - step("clear:filter") else: raise ValueError("Unsupported source_path item: {}".format(claim)) + if build_step: + build_plan.append(build_step) + build_step = [] + self._source_paths = source_paths return build_plan @@ -895,125 +895,145 @@ def execute(self, build_plan, zip_stream, query): sh_work_dir = None pf = None - for action in build_plan: - cmd = action[0] - if cmd.startswith("zip"): - ts = 0 if cmd == "zip:embedded" else None - source_path, prefix = action[1:] - if not sh_work_dir: - sh_work_dir = tf_work_dir - log.debug("WORKDIR: %s", sh_work_dir) - if source_path: - if not os.path.isabs(source_path): - source_path = os.path.normpath( - os.path.join(sh_work_dir, source_path) - ) - else: - source_path = sh_work_dir - if os.path.isdir(source_path): - if pf: - self._zip_write_with_filter( - zs, pf, source_path, prefix, timestamp=ts - ) + for step in build_plan: + # init step + sh_work_dir = tf_work_dir + if pf: + pf.reset() + pf = None + + log.debug("STEPDIR: %s", sh_work_dir) + + # execute step actions + for action in step: + cmd = action[0] + if cmd.startswith("zip"): + ts = 0 if cmd == "zip:embedded" else None + + source_path, prefix = None, None + n = len(action) + if n == 2: + source_path = action[1] + elif n == 3: + source_path, prefix = action[1:] + + if source_path: + if not os.path.isabs(source_path): + source_path = os.path.normpath( + os.path.join(sh_work_dir, source_path) + ) else: - zs.write_dirs(source_path, prefix=prefix, timestamp=ts) - else: - zs.write_file(source_path, prefix=prefix, timestamp=ts) - elif cmd == "pip": - runtime, pip_requirements, prefix, tmp_dir = action[1:] - with install_pip_requirements(query, pip_requirements, tmp_dir) as rd: - if rd: + source_path = sh_work_dir + if os.path.isdir(source_path): if pf: - self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0) - else: - # XXX: timestamp=0 - what actually do with it? - zs.write_dirs(rd, prefix=prefix, timestamp=0) - elif cmd == "poetry": - ( - runtime, - path, - poetry_export_extra_args, - prefix, - ) = action[1:] - log.info("poetry_export_extra_args: %s", poetry_export_extra_args) - with install_poetry_dependencies( - query, path, poetry_export_extra_args - ) as rd: - if rd: - if pf: - self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0) - else: - # XXX: timestamp=0 - what actually do with it? - zs.write_dirs(rd, prefix=prefix, timestamp=0) - elif cmd == "npm": - runtime, npm_requirements, prefix, tmp_dir = action[1:] - with install_npm_requirements(query, npm_requirements, tmp_dir) as rd: - if rd: - if pf: - self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0) + self._zip_write_with_filter( + zs, pf, source_path, prefix, timestamp=ts + ) else: - # XXX: timestamp=0 - what actually do with it? - zs.write_dirs(rd, prefix=prefix, timestamp=0) - elif cmd == "sh": - with tempfile.NamedTemporaryFile(mode="w+t", delete=True) as temp_file: - path, script = action[1:] - - if not path: - path = tf_work_dir - if not os.path.isabs(path): - path = os.path.normpath(os.path.join(tf_work_dir, path)) - - if log.isEnabledFor(DEBUG2): - log.debug("exec shell script ...") - for line in script.splitlines(): - sh_log.debug(line) - - script = "\n".join( - ( - script, - # NOTE: Execute `pwd` to determine the subprocess shell's - # working directory after having executed all other commands. - "retcode=$?", - f"pwd >{temp_file.name}", - "exit $retcode", + zs.write_dirs(source_path, prefix=prefix, timestamp=ts) + else: + zs.write_file(source_path, prefix=prefix, timestamp=ts) + elif cmd == "pip": + runtime, pip_requirements, prefix, tmp_dir = action[1:] + with install_pip_requirements( + query, pip_requirements, tmp_dir + ) as rd: + if rd: + if pf: + self._zip_write_with_filter( + zs, pf, rd, prefix, timestamp=0 + ) + else: + # XXX: timestamp=0 - what actually do with it? + zs.write_dirs(rd, prefix=prefix, timestamp=0) + elif cmd == "poetry": + ( + runtime, + path, + poetry_export_extra_args, + prefix, + ) = action[1:] + log.info("poetry_export_extra_args: %s", poetry_export_extra_args) + with install_poetry_dependencies( + query, path, poetry_export_extra_args + ) as rd: + if rd: + if pf: + self._zip_write_with_filter( + zs, pf, rd, prefix, timestamp=0 + ) + else: + # XXX: timestamp=0 - what actually do with it? + zs.write_dirs(rd, prefix=prefix, timestamp=0) + elif cmd == "npm": + runtime, npm_requirements, prefix, tmp_dir = action[1:] + with install_npm_requirements( + query, npm_requirements, tmp_dir + ) as rd: + if rd: + if pf: + self._zip_write_with_filter( + zs, pf, rd, prefix, timestamp=0 + ) + else: + # XXX: timestamp=0 - what actually do with it? + zs.write_dirs(rd, prefix=prefix, timestamp=0) + elif cmd == "sh": + with tempfile.NamedTemporaryFile( + mode="w+t", delete=True + ) as temp_file: + script = action[1] + + if log.isEnabledFor(DEBUG2): + log.debug("exec shell script ...") + for line in script.splitlines(): + sh_log.debug(line) + + script = "\n".join( + ( + script, + # NOTE: Execute `pwd` to determine the subprocess shell's + # working directory after having executed all other commands. + "retcode=$?", + f"pwd >{temp_file.name}", + "exit $retcode", + ) ) - ) - p = subprocess.Popen( - script, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=path, - ) + p = subprocess.Popen( + script, + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=sh_work_dir, + ) - call_stdout, call_stderr = p.communicate() - exit_code = p.returncode - log.debug("exit_code: %s", exit_code) - if exit_code != 0: - raise RuntimeError( - "Script did not run successfully, exit code {}: {} - {}".format( - exit_code, - call_stdout.decode("utf-8").strip(), - call_stderr.decode("utf-8").strip(), + call_stdout, call_stderr = p.communicate() + exit_code = p.returncode + log.debug("exit_code: %s", exit_code) + if exit_code != 0: + raise RuntimeError( + "Script did not run successfully, exit code {}: {} - {}".format( + exit_code, + call_stdout.decode("utf-8").strip(), + call_stderr.decode("utf-8").strip(), + ) ) - ) - temp_file.seek(0) - # NOTE: This var `sh_work_dir` is consumed in cmd == "zip" loop - sh_work_dir = temp_file.read().strip() + temp_file.seek(0) + # NOTE: This var `sh_work_dir` is consumed in cmd == "zip" loop + sh_work_dir = temp_file.read().strip() + log.debug("WORKDIR: %s", sh_work_dir) + + elif cmd == "set:workdir": + path = action[1] + sh_work_dir = os.path.normpath(os.path.join(tf_work_dir, path)) log.debug("WORKDIR: %s", sh_work_dir) - elif cmd == "reset:workdir": - sh_work_dir = tf_work_dir - log.debug("WORKDIR: %s", sh_work_dir) - elif cmd == "set:filter": - patterns = action[1] - pf = ZipContentFilter(args=self._args) - pf.compile(patterns) - elif cmd == "clear:filter": - pf.reset() - pf = None + elif cmd == "set:filter": + patterns = action[1] + pf = ZipContentFilter(args=self._args) + pf.compile(patterns) @staticmethod def _zip_write_with_filter( @@ -1616,11 +1636,11 @@ def prepare_command(args): content_hash = content_hash.hexdigest() # Generate a unique filename based on the hash. - filename = os.path.join(artifacts_dir, "{}.zip".format(content_hash)) + zip_filename = os.path.join(artifacts_dir, "{}.zip".format(content_hash)) # Compute timestamp trigger was_missing = False - filename_path = os.path.join(os.getcwd(), filename) + filename_path = os.path.join(os.getcwd(), zip_filename) if recreate_missing_package: if os.path.exists(filename_path): st = os.stat(filename_path) @@ -1633,7 +1653,7 @@ def prepare_command(args): # Replace variables in the build command with calculated values. build_data = { - "filename": filename, + "filename": zip_filename, "runtime": runtime, "artifacts_dir": artifacts_dir, "build_plan": build_plan, @@ -1653,7 +1673,7 @@ def prepare_command(args): # Output the result to Terraform. json.dump( { - "filename": filename, + "filename": zip_filename, "build_plan": build_plan, "build_plan_filename": build_plan_filename, "timestamp": str(timestamp), diff --git a/tests/test_package_toml.py b/tests/test_package_toml.py index b60550a4..9eba3f4a 100644 --- a/tests/test_package_toml.py +++ b/tests/test_package_toml.py @@ -26,7 +26,7 @@ def test_build_manager_failing_command(): bpm = BuildPlanManager(args=Mock()) with raises(Exception): bpm.execute( - build_plan=[["sh", "/tmp", "NOTACOMMAND"]], + build_plan=[[["sh", "/tmp", "NOTACOMMAND"]]], zip_stream=None, query=None, ) diff --git a/tests/test_zip_source.py b/tests/test_zip_source.py index f46466e1..dd6750ca 100644 --- a/tests/test_zip_source.py +++ b/tests/test_zip_source.py @@ -12,8 +12,10 @@ def test_zip_source_path_sh_work_dir(): bpm.execute( build_plan=[ - ["sh", ".", "cd $(mktemp -d)\n echo pip install"], - ["zip:embedded", ".", "./python"], + [ + ["sh", "cd $(mktemp -d)\n echo pip install"], + ["zip:embedded", ".", "./python"], + ] ], zip_stream=zs, query=None, @@ -33,8 +35,10 @@ def test_zip_source_path(): bpm.execute( build_plan=[ - ["sh", ".", "echo pip install"], - ["zip:embedded", ".", "./python"], + [ + ["sh", "echo pip install"], + ["zip:embedded", ".", "./python"], + ] ], zip_stream=zs, query=None,