From b68379ed1bb7639b3ab05c172b91adcdb6112699 Mon Sep 17 00:00:00 2001 From: Gilad Gershtein Date: Wed, 23 Mar 2022 16:00:55 +0200 Subject: [PATCH 1/7] don't copy local __pycache__ into the build docker --- lambdabuild/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdabuild/cli.py b/lambdabuild/cli.py index e05be51..cbb474d 100644 --- a/lambdabuild/cli.py +++ b/lambdabuild/cli.py @@ -107,7 +107,7 @@ def delete_excluded_files(build_dir): if dirname in dirs: dirs.remove(dirname) shutil.rmtree(os.path.join(root, dirname)) - for basename in args.exclude_basenames: + for basename in args.exclude_basenames + ['__pycache__']: if basename in files: files.remove(basename) os.unlink(os.path.join(root, basename)) From 1607a9fa6ed2ea8e4f72869f9934787939c6cff3 Mon Sep 17 00:00:00 2001 From: Gilad Gershtein Date: Wed, 23 Mar 2022 16:04:12 +0200 Subject: [PATCH 2/7] bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6d75cc6..2ec8f00 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="lambdabuild", - version="0.2.5", + version="0.2.6", author="Shlomi Matichin", author_email="shlomomatichin@gmail.com", description="Easily build AWS lambda artifacts and layers, correctly", From 3b183dacd23aea5e1e39468676ebd3a96326cf1c Mon Sep 17 00:00:00 2001 From: Gilad Gershtein Date: Tue, 28 Mar 2023 12:52:42 +0300 Subject: [PATCH 3/7] added python 3.9 support --- lambdabuild/byteequivalentzip.py | 3 ++- lambdabuild/clean_lambda_directory_before_zip_python3.py | 7 +++++-- lambdabuild/cli.py | 3 ++- lambdabuild/runtimeinfo.py | 6 ++++++ setup.py | 2 +- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/lambdabuild/byteequivalentzip.py b/lambdabuild/byteequivalentzip.py index 401c81f..66b86c2 100644 --- a/lambdabuild/byteequivalentzip.py +++ b/lambdabuild/byteequivalentzip.py @@ -10,6 +10,7 @@ PYTHON36_HEADER = b"\x33\x0d\x0d\x0a" PYTHON37_HEADER = b"\x42\x0d\x0d\x0a" PYTHON38_HEADER = b"\x55\x0d\x0d\x0a" +PYTHON39_HEADER = b"\x61\x0d\x0d\x0a" PYTHON27_HEADER = b"\x03\xf3\x0d\x0a" @@ -25,7 +26,7 @@ def _add_to_zip(name, contents): context['lastName'] = name if name.endswith(".pyc"): assert len(PYC_UNIX_TIMESTAMP) == 4 - if contents.startswith(PYTHON37_HEADER) or contents.startswith(PYTHON38_HEADER): + if contents.startswith(PYTHON37_HEADER) or contents.startswith(PYTHON38_HEADER) or contents.startswith(PYTHON39_HEADER): contents = contents[:8] + PYC_UNIX_TIMESTAMP + contents[12:] else: assert contents.startswith(PYTHON27_HEADER) or contents.startswith(PYTHON36_HEADER), \ diff --git a/lambdabuild/clean_lambda_directory_before_zip_python3.py b/lambdabuild/clean_lambda_directory_before_zip_python3.py index ba07b8c..fbbb140 100755 --- a/lambdabuild/clean_lambda_directory_before_zip_python3.py +++ b/lambdabuild/clean_lambda_directory_before_zip_python3.py @@ -65,8 +65,11 @@ writer.write(f"import {entry_point}\n") module_finder.run_script("/tmp/entrypoint.py") for module in module_finder.modules.values(): - if module.__file__ is not None and module.__file__.startswith(args.dir): - py_files_to_keep.add(os.path.abspath(module.__file__)) + if not module.__file__: + continue + file_path = os.path.abspath(module.__file__) + if file_path.startswith(os.path.abspath(args.dir)): + py_files_to_keep.add(file_path) never_delete = set(os.path.join(args.dir, p) for p in NEVER_DELETE) for root, dirs, files in os.walk(args.dir): diff --git a/lambdabuild/cli.py b/lambdabuild/cli.py index cbb474d..c78209f 100644 --- a/lambdabuild/cli.py +++ b/lambdabuild/cli.py @@ -14,10 +14,11 @@ from lambdabuild import collectsources parser = argparse.ArgumentParser() -parser.add_argument("--runtime", default="3.6") +parser.add_argument("--runtime", default="3.9") parser.add_argument( "--base-docker", choices=[ + "lambci/lambda:build-python3.9", "lambci/lambda:build-python3.8", "lambci/lambda:build-python3.7", "lambci/lambda:build-python3.6", diff --git a/lambdabuild/runtimeinfo.py b/lambdabuild/runtimeinfo.py index b2d1a91..8338c8f 100644 --- a/lambdabuild/runtimeinfo.py +++ b/lambdabuild/runtimeinfo.py @@ -19,9 +19,15 @@ two_or_three="3", three="3", ), + '3.9': dict( + base_image="public.ecr.aws/sam/build-python3.9", + two_or_three="3", + three="3", + ), } BASE_IMAGES = [ + "public.ecr.aws/sam/build-python3.9", "lambci/lambda:build-python3.8", "lambci/lambda:build-python3.7", "lambci/lambda:build-python3.6", diff --git a/setup.py b/setup.py index 2ec8f00..ca33430 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="lambdabuild", - version="0.2.6", + version="0.2.7", author="Shlomi Matichin", author_email="shlomomatichin@gmail.com", description="Easily build AWS lambda artifacts and layers, correctly", From 26604929d58838354a1fd89618f8e913e2e48587 Mon Sep 17 00:00:00 2001 From: Gilad Gershtein Date: Fri, 31 Mar 2023 18:55:42 +0300 Subject: [PATCH 4/7] don't copy local __pycache__ --- lambdabuild/cli.py | 4 ++-- lambdabuild/dockerfiletemplates.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lambdabuild/cli.py b/lambdabuild/cli.py index c78209f..3929aa1 100644 --- a/lambdabuild/cli.py +++ b/lambdabuild/cli.py @@ -104,11 +104,11 @@ def create_build_dir(): def delete_excluded_files(build_dir): regexes = [re.compile(r) for r in args.exclude_regexes] for root, dirs, files in os.walk(build_dir): - for dirname in args.exclude_dirs: + for dirname in args.exclude_dirs + ['__pycache__']: if dirname in dirs: dirs.remove(dirname) shutil.rmtree(os.path.join(root, dirname)) - for basename in args.exclude_basenames + ['__pycache__']: + for basename in args.exclude_basenames: if basename in files: files.remove(basename) os.unlink(os.path.join(root, basename)) diff --git a/lambdabuild/dockerfiletemplates.py b/lambdabuild/dockerfiletemplates.py index eff5d15..8ca9f92 100644 --- a/lambdabuild/dockerfiletemplates.py +++ b/lambdabuild/dockerfiletemplates.py @@ -21,6 +21,7 @@ WORKDIR /lambdabuild COPY clean_lambda_directory_before_zip_python%(two_or_three)s.py / COPY sourcecode/ /lambdabuild/ +RUN find /lambdabuild/ -name '__pycache__' | xargs rm -rf RUN python%(three)s /clean_lambda_directory_before_zip_python%(two_or_three)s.py . %(entry_points)s COPY rawfiles/ /lambdabuild/ RUN find -name '*.so*' | xargs -I @ strip @ From 6d784b6485a997231df0419cdb8d66b1cef15658 Mon Sep 17 00:00:00 2001 From: Gilad Gershtein Date: Mon, 15 Sep 2025 11:46:36 +0300 Subject: [PATCH 5/7] added py3.13 support --- lambdabuild/byteequivalentzip.py | 5 +++-- lambdabuild/cli.py | 3 ++- lambdabuild/runtimeinfo.py | 6 ++++++ setup.py | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lambdabuild/byteequivalentzip.py b/lambdabuild/byteequivalentzip.py index 66b86c2..824ebaa 100644 --- a/lambdabuild/byteequivalentzip.py +++ b/lambdabuild/byteequivalentzip.py @@ -11,6 +11,7 @@ PYTHON37_HEADER = b"\x42\x0d\x0d\x0a" PYTHON38_HEADER = b"\x55\x0d\x0d\x0a" PYTHON39_HEADER = b"\x61\x0d\x0d\x0a" +PYTHON313_HEADER = b"\xf3\x0d\x0d\x0a" PYTHON27_HEADER = b"\x03\xf3\x0d\x0a" @@ -26,11 +27,11 @@ def _add_to_zip(name, contents): context['lastName'] = name if name.endswith(".pyc"): assert len(PYC_UNIX_TIMESTAMP) == 4 - if contents.startswith(PYTHON37_HEADER) or contents.startswith(PYTHON38_HEADER) or contents.startswith(PYTHON39_HEADER): + if contents.startswith(PYTHON37_HEADER) or contents.startswith(PYTHON38_HEADER) or contents.startswith(PYTHON39_HEADER) or contents.startswith(PYTHON313_HEADER): contents = contents[:8] + PYC_UNIX_TIMESTAMP + contents[12:] else: assert contents.startswith(PYTHON27_HEADER) or contents.startswith(PYTHON36_HEADER), \ - "Unrecognized pyc header, are you using python 2.7, 3.6 or 3.7? %s" % contents[:4] + "Unrecognized pyc header, are you using python 2.7, 3.6, 3.7, 3.8, 3.9 or 3.13? %s" % contents[:4] contents = contents[:4] + PYC_UNIX_TIMESTAMP + contents[8:] info = zipfile.ZipInfo(filename=name, date_time=(1980, 1, 1, 0, 0, 0)) info.external_attr |= (0x1a4 << 16) diff --git a/lambdabuild/cli.py b/lambdabuild/cli.py index 3929aa1..ec3d022 100644 --- a/lambdabuild/cli.py +++ b/lambdabuild/cli.py @@ -14,10 +14,11 @@ from lambdabuild import collectsources parser = argparse.ArgumentParser() -parser.add_argument("--runtime", default="3.9") +parser.add_argument("--runtime", default="3.13") parser.add_argument( "--base-docker", choices=[ + "public.ecr.aws/sam/build-python3.13", "lambci/lambda:build-python3.9", "lambci/lambda:build-python3.8", "lambci/lambda:build-python3.7", diff --git a/lambdabuild/runtimeinfo.py b/lambdabuild/runtimeinfo.py index 8338c8f..5939783 100644 --- a/lambdabuild/runtimeinfo.py +++ b/lambdabuild/runtimeinfo.py @@ -24,9 +24,15 @@ two_or_three="3", three="3", ), + '3.13': dict( + base_image="public.ecr.aws/sam/build-python3.13", + two_or_three="3", + three="3", + ), } BASE_IMAGES = [ + "public.ecr.aws/sam/build-python3.13", "public.ecr.aws/sam/build-python3.9", "lambci/lambda:build-python3.8", "lambci/lambda:build-python3.7", diff --git a/setup.py b/setup.py index ca33430..a4d4bdc 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="lambdabuild", - version="0.2.7", + version="0.2.8", author="Shlomi Matichin", author_email="shlomomatichin@gmail.com", description="Easily build AWS lambda artifacts and layers, correctly", From 134c85a98b43f49bd85ddf810851c146ec0900d7 Mon Sep 17 00:00:00 2001 From: Gilad Gershtein Date: Mon, 15 Sep 2025 14:24:09 +0300 Subject: [PATCH 6/7] added py3.11 --- lambdabuild/byteequivalentzip.py | 5 +++-- lambdabuild/cli.py | 1 + lambdabuild/runtimeinfo.py | 5 +++++ setup.py | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lambdabuild/byteequivalentzip.py b/lambdabuild/byteequivalentzip.py index 824ebaa..54966d5 100644 --- a/lambdabuild/byteequivalentzip.py +++ b/lambdabuild/byteequivalentzip.py @@ -11,6 +11,7 @@ PYTHON37_HEADER = b"\x42\x0d\x0d\x0a" PYTHON38_HEADER = b"\x55\x0d\x0d\x0a" PYTHON39_HEADER = b"\x61\x0d\x0d\x0a" +PYTHON311_HEADER = b"\xa7\x0d\x0d\x0a" PYTHON313_HEADER = b"\xf3\x0d\x0d\x0a" PYTHON27_HEADER = b"\x03\xf3\x0d\x0a" @@ -27,11 +28,11 @@ def _add_to_zip(name, contents): context['lastName'] = name if name.endswith(".pyc"): assert len(PYC_UNIX_TIMESTAMP) == 4 - if contents.startswith(PYTHON37_HEADER) or contents.startswith(PYTHON38_HEADER) or contents.startswith(PYTHON39_HEADER) or contents.startswith(PYTHON313_HEADER): + if contents.startswith(PYTHON37_HEADER) or contents.startswith(PYTHON38_HEADER) or contents.startswith(PYTHON39_HEADER) or contents.startswith(PYTHON311_HEADER) or contents.startswith(PYTHON313_HEADER): contents = contents[:8] + PYC_UNIX_TIMESTAMP + contents[12:] else: assert contents.startswith(PYTHON27_HEADER) or contents.startswith(PYTHON36_HEADER), \ - "Unrecognized pyc header, are you using python 2.7, 3.6, 3.7, 3.8, 3.9 or 3.13? %s" % contents[:4] + "Unrecognized pyc header, are you using python 2.7, 3.6, 3.7, 3.8, 3.9, 3.11 or 3.13? %s" % contents[:4] contents = contents[:4] + PYC_UNIX_TIMESTAMP + contents[8:] info = zipfile.ZipInfo(filename=name, date_time=(1980, 1, 1, 0, 0, 0)) info.external_attr |= (0x1a4 << 16) diff --git a/lambdabuild/cli.py b/lambdabuild/cli.py index ec3d022..3c6f2c2 100644 --- a/lambdabuild/cli.py +++ b/lambdabuild/cli.py @@ -19,6 +19,7 @@ "--base-docker", choices=[ "public.ecr.aws/sam/build-python3.13", + "public.ecr.aws/sam/build-python3.11", "lambci/lambda:build-python3.9", "lambci/lambda:build-python3.8", "lambci/lambda:build-python3.7", diff --git a/lambdabuild/runtimeinfo.py b/lambdabuild/runtimeinfo.py index 5939783..a1de7a3 100644 --- a/lambdabuild/runtimeinfo.py +++ b/lambdabuild/runtimeinfo.py @@ -24,6 +24,11 @@ two_or_three="3", three="3", ), + '3.11': dict( + base_image="public.ecr.aws/sam/build-python3.11", + two_or_three="3", + three="3", + ), '3.13': dict( base_image="public.ecr.aws/sam/build-python3.13", two_or_three="3", diff --git a/setup.py b/setup.py index a4d4bdc..9db9af4 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="lambdabuild", - version="0.2.8", + version="0.2.10", author="Shlomi Matichin", author_email="shlomomatichin@gmail.com", description="Easily build AWS lambda artifacts and layers, correctly", From 70262bfbb5576f3a38e8ca88769ab016d0a76e71 Mon Sep 17 00:00:00 2001 From: Gilad Gershtein Date: Tue, 16 Sep 2025 13:25:59 +0300 Subject: [PATCH 7/7] remove pycs --- .../clean_lambda_directory_before_zip_python2.py | 4 ++++ .../clean_lambda_directory_before_zip_python3.py | 16 ++++++++++++++++ setup.py | 2 +- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lambdabuild/clean_lambda_directory_before_zip_python2.py b/lambdabuild/clean_lambda_directory_before_zip_python2.py index b76c3e0..ee41fe5 100755 --- a/lambdabuild/clean_lambda_directory_before_zip_python2.py +++ b/lambdabuild/clean_lambda_directory_before_zip_python2.py @@ -87,6 +87,10 @@ py_compile.compile(full_path) if args.sourceless: os.unlink(full_path) + else: + # When keeping sources, remove any existing .pyc files + if os.path.exists(pyc): + os.unlink(pyc) for dirname in list(dirs): for extension in DIRECTORIES_EXTENSIONS_TO_DELETE: if dirname.endswith(extension): diff --git a/lambdabuild/clean_lambda_directory_before_zip_python3.py b/lambdabuild/clean_lambda_directory_before_zip_python3.py index fbbb140..64d56fd 100755 --- a/lambdabuild/clean_lambda_directory_before_zip_python3.py +++ b/lambdabuild/clean_lambda_directory_before_zip_python3.py @@ -94,6 +94,12 @@ if args.sourceless: if full_path not in never_delete: os.unlink(full_path) + else: + # When keeping sources, remove any existing .pyc files + pyc_path = full_path + "c" + if os.path.exists(pyc_path): + os.unlink(pyc_path) + if args.sourceless: for root, dirs, unused_files in os.walk(args.dir): if '__pycache__' not in dirs: @@ -105,3 +111,13 @@ if removeMagic[:-1] not in never_delete: os.rename(filename, os.path.join(root, removeMagic)) os.rmdir(cache) +else: + # When keeping sources, remove all __pycache__ directories and .pyc files + for root, dirs, files in os.walk(args.dir): + for filename in files: + if filename.endswith(".pyc"): + os.unlink(os.path.join(root, filename)) + for dirname in list(dirs): + if dirname == '__pycache__': + shutil.rmtree(os.path.join(root, dirname)) + dirs.remove(dirname) diff --git a/setup.py b/setup.py index 9db9af4..51db79b 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="lambdabuild", - version="0.2.10", + version="0.2.11", author="Shlomi Matichin", author_email="shlomomatichin@gmail.com", description="Easily build AWS lambda artifacts and layers, correctly",