From da32d1bddcea6d169936dacc686278db385bcb98 Mon Sep 17 00:00:00 2001 From: David Souther Date: Wed, 8 Oct 2025 16:21:44 -0400 Subject: [PATCH 1/3] Add -tools build command to modularize internal build processing. --- aws_doc_sdk_examples_tools/builder.py | 95 ++++++++++++++++++++++ aws_doc_sdk_examples_tools/builder_test.py | 0 aws_doc_sdk_examples_tools/fs.py | 12 +++ aws_doc_sdk_examples_tools/snippets.py | 8 +- 4 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 aws_doc_sdk_examples_tools/builder.py create mode 100644 aws_doc_sdk_examples_tools/builder_test.py diff --git a/aws_doc_sdk_examples_tools/builder.py b/aws_doc_sdk_examples_tools/builder.py new file mode 100644 index 0000000..97db224 --- /dev/null +++ b/aws_doc_sdk_examples_tools/builder.py @@ -0,0 +1,95 @@ +from dataclasses import dataclass, field +import logging +from pathlib import Path +import re +from typing import List, Optional, cast + +from aws_doc_sdk_examples_tools.doc_gen import DocGen +from aws_doc_sdk_examples_tools.fs import Fs, PathFs +from aws_doc_sdk_examples_tools.snippets import write_snippets + +logging.basicConfig(level=logging.INFO) + +logger = logging.getLogger(Path(__file__).name) + + +IAM_PATTERN = re.compile(r'"Version"\s*:\s*"20(08|12)-10-17"\s*,') +IAM_PATTERN_SLASHES = re.compile(r'\\"Version\\"\s*:\s*\\"20(08|12)-10-17\\"\s*,') +# This exciting set of unicode characters is the expanded version of the &IAM-2025-waiver; entity. +IAM_WAIVER = '"Version":"2012-10-17",\u0009\u0009\u0020\u0009\u0020\u0009\u0020' +IAM_WAIVER_SLASHES = '\\"Version\\":\\"2012-10-17\\",\u0009\u0009\u0020\u0009\u0020\u0009\u0020' + + +def _iam_replace_all(source: Optional[str]): + if source: + return IAM_PATTERN_SLASHES.subn(IAM_WAIVER_SLASHES, IAM_PATTERN.subn(IAM_WAIVER, source)[0])[0] + return None + + +def _iam_fixup_metadata(meta_folder: Path): + for meta_path in meta_folder.glob("**/*_metadata.yaml"): + with meta_path.open("r") as meta_file: + contents = meta_file.read() + contents = cast(str, _iam_replace_all(contents)) + with meta_path.open("w") as meta_file: + meta_file.write(contents) + + +def _iam_fixup_docgen(doc_gen: DocGen) -> DocGen: + # For performance, do this mutably, but keep the signature open to making DocGen frozen + for snippet in doc_gen.snippets.values(): + snippet.code = cast(str, _iam_replace_all(snippet.code)) + return doc_gen + + +@dataclass +class Builder: + root: Path + dest: Path + doc_gen_folder = ".doc_gen" + snippet_files_folder = "snippet_files" + fs: Fs = field(default_factory=PathFs) + _doc_gen: DocGen = field(init=False) + + def __post_init__(self): + self._doc_gen = DocGen.from_root(self.root) + + def copy_doc_gen(self): + tmp_mirror_doc_gen = self.root / self.doc_gen_folder + mirror_doc_gen = self.dest / self.doc_gen_folder + logger.info(f"Moving cloned files into package from {tmp_mirror_doc_gen} to {mirror_doc_gen}") + try: + self.fs.copytree(tmp_mirror_doc_gen, mirror_doc_gen) + except Exception as e: + logger.error(f"Failed copy directory {tmp_mirror_doc_gen} to {mirror_doc_gen}\n{e}") + logger.error(e) + raise + + def write_snippets(self): + write_snippets(self.dest / self.doc_gen_folder / self.snippet_files_folder, self._doc_gen.snippets) + + def run(self): + logger.debug("Copying docgen files...") + self.copy_doc_gen() + _iam_fixup_metadata(self.dest) + logger.debug("Collecting snippets...") + self._doc_gen.collect_snippets() + self._doc_gen = _iam_fixup_docgen(self._doc_gen) + logger.debug("Writing snippets...") + self.write_snippets() + + +def main(): + from argparse import ArgumentParser + argparse = ArgumentParser() + argparse.add_argument('--root', type=Path, default=Path()) + argparse.add_argument('--dest', type=Path) + # Load a config from somewhere to get doc_gen_folder and snippets_files_folder + args = argparse.parse_args() + + builder = Builder(args.root, args.dest) + builder.run() + + +if __name__ == "__main__": + main() diff --git a/aws_doc_sdk_examples_tools/builder_test.py b/aws_doc_sdk_examples_tools/builder_test.py new file mode 100644 index 0000000..e69de29 diff --git a/aws_doc_sdk_examples_tools/fs.py b/aws_doc_sdk_examples_tools/fs.py index e06e6e6..9b2fceb 100644 --- a/aws_doc_sdk_examples_tools/fs.py +++ b/aws_doc_sdk_examples_tools/fs.py @@ -3,6 +3,7 @@ from fnmatch import fnmatch from os import listdir from pathlib import Path +import shutil from stat import S_ISREG from typing import Dict, Generator, List @@ -47,6 +48,9 @@ def mkdir(self, path: Path): def list(self, path: Path) -> List[Path]: pass + @abstractmethod + def copytree(self, source: Path, dest: Path): + pass class PathFs(Fs): def glob(self, path: Path, glob: str) -> Generator[Path, None, None]: @@ -78,6 +82,9 @@ def list(self, path: Path) -> List[Path]: if self.stat(path).is_file: return [] return [path / name for name in listdir(path)] + + def copytree(self, source: Path, dest: Path): + shutil.copytree(source, dest) class RecordFs(Fs): @@ -137,5 +144,10 @@ def list(self, path: Path) -> List[Path]: return sorted(children) + def copytree(self, source: Path, dest: Path): + for child in self.list(source): + path = child.relative_to(dest).absolute() + self.fs[path] = self.fs[child] + fs = PathFs() diff --git a/aws_doc_sdk_examples_tools/snippets.py b/aws_doc_sdk_examples_tools/snippets.py index 85ea070..96410f5 100644 --- a/aws_doc_sdk_examples_tools/snippets.py +++ b/aws_doc_sdk_examples_tools/snippets.py @@ -17,6 +17,7 @@ ValidationConfig, ) from . import validator_config +from aws_doc_sdk_examples_tools import fs SNIPPET_START = "snippet-start:[" SNIPPET_END = "snippet-end:[" @@ -309,16 +310,15 @@ def validate_snippets( verify_no_secret_keys(snippet.code, root / snippet.file, validation, errors) -def write_snippets(root: Path, snippets: Dict[str, Snippet], check: bool = False): +def write_snippets(root: Path, snippets: Dict[str, Snippet], check: bool = False, fs: Fs = fs.fs): errors = MetadataErrors() for tag in snippets: name = root / f"{tag}.txt" - if check and name.exists(): + if check and fs.stat(name).exists: errors.append(SnippetAlreadyWritten(file=name)) else: try: - with open(name, "w", encoding="utf-8") as file: - file.write(snippets[tag].code) + fs.write(name, snippets[tag].code) except Exception as error: errors.append(SnippetWriteError(file=name, error=error)) return errors From 04dfd3eb1d3e355158a804b0ba02e4dd51c82062 Mon Sep 17 00:00:00 2001 From: David Souther Date: Thu, 9 Oct 2025 14:30:54 -0400 Subject: [PATCH 2/3] Remove IAM things, they will stay internal. --- aws_doc_sdk_examples_tools/builder.py | 33 --------------------------- 1 file changed, 33 deletions(-) diff --git a/aws_doc_sdk_examples_tools/builder.py b/aws_doc_sdk_examples_tools/builder.py index 97db224..7cd1bd3 100644 --- a/aws_doc_sdk_examples_tools/builder.py +++ b/aws_doc_sdk_examples_tools/builder.py @@ -1,8 +1,6 @@ from dataclasses import dataclass, field import logging from pathlib import Path -import re -from typing import List, Optional, cast from aws_doc_sdk_examples_tools.doc_gen import DocGen from aws_doc_sdk_examples_tools.fs import Fs, PathFs @@ -13,35 +11,6 @@ logger = logging.getLogger(Path(__file__).name) -IAM_PATTERN = re.compile(r'"Version"\s*:\s*"20(08|12)-10-17"\s*,') -IAM_PATTERN_SLASHES = re.compile(r'\\"Version\\"\s*:\s*\\"20(08|12)-10-17\\"\s*,') -# This exciting set of unicode characters is the expanded version of the &IAM-2025-waiver; entity. -IAM_WAIVER = '"Version":"2012-10-17",\u0009\u0009\u0020\u0009\u0020\u0009\u0020' -IAM_WAIVER_SLASHES = '\\"Version\\":\\"2012-10-17\\",\u0009\u0009\u0020\u0009\u0020\u0009\u0020' - - -def _iam_replace_all(source: Optional[str]): - if source: - return IAM_PATTERN_SLASHES.subn(IAM_WAIVER_SLASHES, IAM_PATTERN.subn(IAM_WAIVER, source)[0])[0] - return None - - -def _iam_fixup_metadata(meta_folder: Path): - for meta_path in meta_folder.glob("**/*_metadata.yaml"): - with meta_path.open("r") as meta_file: - contents = meta_file.read() - contents = cast(str, _iam_replace_all(contents)) - with meta_path.open("w") as meta_file: - meta_file.write(contents) - - -def _iam_fixup_docgen(doc_gen: DocGen) -> DocGen: - # For performance, do this mutably, but keep the signature open to making DocGen frozen - for snippet in doc_gen.snippets.values(): - snippet.code = cast(str, _iam_replace_all(snippet.code)) - return doc_gen - - @dataclass class Builder: root: Path @@ -71,10 +40,8 @@ def write_snippets(self): def run(self): logger.debug("Copying docgen files...") self.copy_doc_gen() - _iam_fixup_metadata(self.dest) logger.debug("Collecting snippets...") self._doc_gen.collect_snippets() - self._doc_gen = _iam_fixup_docgen(self._doc_gen) logger.debug("Writing snippets...") self.write_snippets() From cea08761bd03039ae2f8191a18733405cf6f19f8 Mon Sep 17 00:00:00 2001 From: David Souther Date: Tue, 14 Oct 2025 16:32:52 +0000 Subject: [PATCH 3/3] Changes locally testing --- .python-version | 2 +- aws_doc_sdk_examples_tools/builder.py | 2 +- aws_doc_sdk_examples_tools/fs.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.python-version b/.python-version index cc1923a..bd28b9c 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.8 +3.9 diff --git a/aws_doc_sdk_examples_tools/builder.py b/aws_doc_sdk_examples_tools/builder.py index 7cd1bd3..04d1b52 100644 --- a/aws_doc_sdk_examples_tools/builder.py +++ b/aws_doc_sdk_examples_tools/builder.py @@ -35,7 +35,7 @@ def copy_doc_gen(self): raise def write_snippets(self): - write_snippets(self.dest / self.doc_gen_folder / self.snippet_files_folder, self._doc_gen.snippets) + write_snippets(self.dest / self.doc_gen_folder / self.snippet_files_folder, self._doc_gen.snippets, self.fs) def run(self): logger.debug("Copying docgen files...") diff --git a/aws_doc_sdk_examples_tools/fs.py b/aws_doc_sdk_examples_tools/fs.py index 9b2fceb..8bad6e4 100644 --- a/aws_doc_sdk_examples_tools/fs.py +++ b/aws_doc_sdk_examples_tools/fs.py @@ -52,6 +52,7 @@ def list(self, path: Path) -> List[Path]: def copytree(self, source: Path, dest: Path): pass + class PathFs(Fs): def glob(self, path: Path, glob: str) -> Generator[Path, None, None]: return path.glob(glob) @@ -65,6 +66,7 @@ def readlines(self, path: Path, encoding: str = "utf-8") -> List[str]: return file.readlines() def write(self, path: Path, content: str): + self.mkdir(path.parent) with path.open("w", encoding="utf-8") as file: file.write(content) @@ -82,9 +84,9 @@ def list(self, path: Path) -> List[Path]: if self.stat(path).is_file: return [] return [path / name for name in listdir(path)] - + def copytree(self, source: Path, dest: Path): - shutil.copytree(source, dest) + shutil.copytree(source, dest, dirs_exist_ok=True) class RecordFs(Fs):