diff --git a/lambdabuild/byteequivalentzip.py b/lambdabuild/byteequivalentzip.py index 401c81f..54966d5 100644 --- a/lambdabuild/byteequivalentzip.py +++ b/lambdabuild/byteequivalentzip.py @@ -10,6 +10,9 @@ PYTHON36_HEADER = b"\x33\x0d\x0d\x0a" PYTHON37_HEADER = b"\x42\x0d\x0d\x0a" PYTHON38_HEADER = b"\x55\x0d\x0d\x0a" +PYTHON39_HEADER = b"\x61\x0d\x0d\x0a" +PYTHON311_HEADER = b"\xa7\x0d\x0d\x0a" +PYTHON313_HEADER = b"\xf3\x0d\x0d\x0a" PYTHON27_HEADER = b"\x03\xf3\x0d\x0a" @@ -25,11 +28,11 @@ def _add_to_zip(name, contents): context['lastName'] = name if name.endswith(".pyc"): assert len(PYC_UNIX_TIMESTAMP) == 4 - if contents.startswith(PYTHON37_HEADER) or contents.startswith(PYTHON38_HEADER): + if contents.startswith(PYTHON37_HEADER) or contents.startswith(PYTHON38_HEADER) or contents.startswith(PYTHON39_HEADER) or contents.startswith(PYTHON311_HEADER) or contents.startswith(PYTHON313_HEADER): contents = contents[:8] + PYC_UNIX_TIMESTAMP + contents[12:] else: assert contents.startswith(PYTHON27_HEADER) or contents.startswith(PYTHON36_HEADER), \ - "Unrecognized pyc header, are you using python 2.7, 3.6 or 3.7? %s" % contents[:4] + "Unrecognized pyc header, are you using python 2.7, 3.6, 3.7, 3.8, 3.9, 3.11 or 3.13? %s" % contents[:4] contents = contents[:4] + PYC_UNIX_TIMESTAMP + contents[8:] info = zipfile.ZipInfo(filename=name, date_time=(1980, 1, 1, 0, 0, 0)) info.external_attr |= (0x1a4 << 16) diff --git a/lambdabuild/clean_lambda_directory_before_zip_python2.py b/lambdabuild/clean_lambda_directory_before_zip_python2.py index b76c3e0..ee41fe5 100755 --- a/lambdabuild/clean_lambda_directory_before_zip_python2.py +++ b/lambdabuild/clean_lambda_directory_before_zip_python2.py @@ -87,6 +87,10 @@ py_compile.compile(full_path) if args.sourceless: os.unlink(full_path) + else: + # When keeping sources, remove any existing .pyc files + if os.path.exists(pyc): + os.unlink(pyc) for dirname in list(dirs): for extension in DIRECTORIES_EXTENSIONS_TO_DELETE: if dirname.endswith(extension): diff --git a/lambdabuild/clean_lambda_directory_before_zip_python3.py b/lambdabuild/clean_lambda_directory_before_zip_python3.py index ba07b8c..64d56fd 100755 --- a/lambdabuild/clean_lambda_directory_before_zip_python3.py +++ b/lambdabuild/clean_lambda_directory_before_zip_python3.py @@ -65,8 +65,11 @@ writer.write(f"import {entry_point}\n") module_finder.run_script("/tmp/entrypoint.py") for module in module_finder.modules.values(): - if module.__file__ is not None and module.__file__.startswith(args.dir): - py_files_to_keep.add(os.path.abspath(module.__file__)) + if not module.__file__: + continue + file_path = os.path.abspath(module.__file__) + if file_path.startswith(os.path.abspath(args.dir)): + py_files_to_keep.add(file_path) never_delete = set(os.path.join(args.dir, p) for p in NEVER_DELETE) for root, dirs, files in os.walk(args.dir): @@ -91,6 +94,12 @@ if args.sourceless: if full_path not in never_delete: os.unlink(full_path) + else: + # When keeping sources, remove any existing .pyc files + pyc_path = full_path + "c" + if os.path.exists(pyc_path): + os.unlink(pyc_path) + if args.sourceless: for root, dirs, unused_files in os.walk(args.dir): if '__pycache__' not in dirs: @@ -102,3 +111,13 @@ if removeMagic[:-1] not in never_delete: os.rename(filename, os.path.join(root, removeMagic)) os.rmdir(cache) +else: + # When keeping sources, remove all __pycache__ directories and .pyc files + for root, dirs, files in os.walk(args.dir): + for filename in files: + if filename.endswith(".pyc"): + os.unlink(os.path.join(root, filename)) + for dirname in list(dirs): + if dirname == '__pycache__': + shutil.rmtree(os.path.join(root, dirname)) + dirs.remove(dirname) diff --git a/lambdabuild/cli.py b/lambdabuild/cli.py index e05be51..3c6f2c2 100644 --- a/lambdabuild/cli.py +++ b/lambdabuild/cli.py @@ -14,10 +14,13 @@ from lambdabuild import collectsources parser = argparse.ArgumentParser() -parser.add_argument("--runtime", default="3.6") +parser.add_argument("--runtime", default="3.13") parser.add_argument( "--base-docker", choices=[ + "public.ecr.aws/sam/build-python3.13", + "public.ecr.aws/sam/build-python3.11", + "lambci/lambda:build-python3.9", "lambci/lambda:build-python3.8", "lambci/lambda:build-python3.7", "lambci/lambda:build-python3.6", @@ -103,7 +106,7 @@ def create_build_dir(): def delete_excluded_files(build_dir): regexes = [re.compile(r) for r in args.exclude_regexes] for root, dirs, files in os.walk(build_dir): - for dirname in args.exclude_dirs: + for dirname in args.exclude_dirs + ['__pycache__']: if dirname in dirs: dirs.remove(dirname) shutil.rmtree(os.path.join(root, dirname)) diff --git a/lambdabuild/dockerfiletemplates.py b/lambdabuild/dockerfiletemplates.py index eff5d15..8ca9f92 100644 --- a/lambdabuild/dockerfiletemplates.py +++ b/lambdabuild/dockerfiletemplates.py @@ -21,6 +21,7 @@ WORKDIR /lambdabuild COPY clean_lambda_directory_before_zip_python%(two_or_three)s.py / COPY sourcecode/ /lambdabuild/ +RUN find /lambdabuild/ -name '__pycache__' | xargs rm -rf RUN python%(three)s /clean_lambda_directory_before_zip_python%(two_or_three)s.py . %(entry_points)s COPY rawfiles/ /lambdabuild/ RUN find -name '*.so*' | xargs -I @ strip @ diff --git a/lambdabuild/runtimeinfo.py b/lambdabuild/runtimeinfo.py index b2d1a91..a1de7a3 100644 --- a/lambdabuild/runtimeinfo.py +++ b/lambdabuild/runtimeinfo.py @@ -19,9 +19,26 @@ two_or_three="3", three="3", ), + '3.9': dict( + base_image="public.ecr.aws/sam/build-python3.9", + two_or_three="3", + three="3", + ), + '3.11': dict( + base_image="public.ecr.aws/sam/build-python3.11", + two_or_three="3", + three="3", + ), + '3.13': dict( + base_image="public.ecr.aws/sam/build-python3.13", + two_or_three="3", + three="3", + ), } BASE_IMAGES = [ + "public.ecr.aws/sam/build-python3.13", + "public.ecr.aws/sam/build-python3.9", "lambci/lambda:build-python3.8", "lambci/lambda:build-python3.7", "lambci/lambda:build-python3.6", diff --git a/setup.py b/setup.py index 6d75cc6..51db79b 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="lambdabuild", - version="0.2.5", + version="0.2.11", author="Shlomi Matichin", author_email="shlomomatichin@gmail.com", description="Easily build AWS lambda artifacts and layers, correctly",