Local pyc shouldn't be copied to the build docker

lambdabuild/byteequivalentzip.py

@@ -10,6 +10,9 @@
 PYTHON36_HEADER = b"\x33\x0d\x0d\x0a"
 PYTHON37_HEADER = b"\x42\x0d\x0d\x0a"
 PYTHON38_HEADER = b"\x55\x0d\x0d\x0a"
+PYTHON39_HEADER = b"\x61\x0d\x0d\x0a"
+PYTHON311_HEADER = b"\xa7\x0d\x0d\x0a"
+PYTHON313_HEADER = b"\xf3\x0d\x0d\x0a"
 PYTHON27_HEADER = b"\x03\xf3\x0d\x0a"
 
 
@@ -25,11 +28,11 @@ def _add_to_zip(name, contents):
             context['lastName'] = name
             if name.endswith(".pyc"):
                 assert len(PYC_UNIX_TIMESTAMP) == 4
-                if contents.startswith(PYTHON37_HEADER) or contents.startswith(PYTHON38_HEADER):
+                if contents.startswith(PYTHON37_HEADER) or contents.startswith(PYTHON38_HEADER) or contents.startswith(PYTHON39_HEADER) or contents.startswith(PYTHON311_HEADER) or contents.startswith(PYTHON313_HEADER):
                     contents = contents[:8] + PYC_UNIX_TIMESTAMP + contents[12:]
                 else:
                     assert contents.startswith(PYTHON27_HEADER) or contents.startswith(PYTHON36_HEADER), \
-                            "Unrecognized pyc header, are you using python 2.7, 3.6 or 3.7? %s" % contents[:4]
+                            "Unrecognized pyc header, are you using python 2.7, 3.6, 3.7, 3.8, 3.9, 3.11 or 3.13? %s" % contents[:4]
                     contents = contents[:4] + PYC_UNIX_TIMESTAMP + contents[8:]
             info = zipfile.ZipInfo(filename=name, date_time=(1980, 1, 1, 0, 0, 0))
             info.external_attr |= (0x1a4 << 16)

lambdabuild/clean_lambda_directory_before_zip_python2.py

@@ -87,6 +87,10 @@
                     py_compile.compile(full_path)
             if args.sourceless:
                 os.unlink(full_path)
+            else:
+                # When keeping sources, remove any existing .pyc files
+                if os.path.exists(pyc):
+                    os.unlink(pyc)
     for dirname in list(dirs):
         for extension in DIRECTORIES_EXTENSIONS_TO_DELETE:
             if dirname.endswith(extension):

lambdabuild/clean_lambda_directory_before_zip_python3.py

@@ -65,8 +65,11 @@
             writer.write(f"import {entry_point}\n")
         module_finder.run_script("/tmp/entrypoint.py")
     for module in module_finder.modules.values():
-        if module.__file__ is not None and module.__file__.startswith(args.dir):
-            py_files_to_keep.add(os.path.abspath(module.__file__))
+        if not module.__file__:
+            continue
+        file_path = os.path.abspath(module.__file__)
+        if file_path.startswith(os.path.abspath(args.dir)):
+            py_files_to_keep.add(file_path)
 
 never_delete = set(os.path.join(args.dir, p) for p in NEVER_DELETE)
 for root, dirs, files in os.walk(args.dir):
@@ -91,6 +94,12 @@
             if args.sourceless:
                 if full_path not in never_delete:
                     os.unlink(full_path)
+            else:
+                # When keeping sources, remove any existing .pyc files
+                pyc_path = full_path + "c"
+                if os.path.exists(pyc_path):
+                    os.unlink(pyc_path)
+
 if args.sourceless:
     for root, dirs, unused_files in os.walk(args.dir):
         if '__pycache__' not in dirs:
@@ -102,3 +111,13 @@
             if removeMagic[:-1] not in never_delete:
                 os.rename(filename, os.path.join(root, removeMagic))
         os.rmdir(cache)
+else:
+    # When keeping sources, remove all __pycache__ directories and .pyc files
+    for root, dirs, files in os.walk(args.dir):
+        for filename in files:
+            if filename.endswith(".pyc"):
+                os.unlink(os.path.join(root, filename))
+        for dirname in list(dirs):
+            if dirname == '__pycache__':
+                shutil.rmtree(os.path.join(root, dirname))
+                dirs.remove(dirname)

lambdabuild/cli.py

@@ -14,10 +14,13 @@
 from lambdabuild import collectsources
 
 parser = argparse.ArgumentParser()
-parser.add_argument("--runtime", default="3.6")
+parser.add_argument("--runtime", default="3.13")
 parser.add_argument(
     "--base-docker",
     choices=[
+        "public.ecr.aws/sam/build-python3.13",
+        "public.ecr.aws/sam/build-python3.11",
+        "lambci/lambda:build-python3.9",
         "lambci/lambda:build-python3.8",
         "lambci/lambda:build-python3.7",
         "lambci/lambda:build-python3.6",
@@ -103,7 +106,7 @@ def create_build_dir():
 def delete_excluded_files(build_dir):
     regexes = [re.compile(r) for r in args.exclude_regexes]
     for root, dirs, files in os.walk(build_dir):
-        for dirname in args.exclude_dirs:
+        for dirname in args.exclude_dirs + ['__pycache__']:
             if dirname in dirs:
                 dirs.remove(dirname)
                 shutil.rmtree(os.path.join(root, dirname))

lambdabuild/dockerfiletemplates.py

@@ -21,6 +21,7 @@
 WORKDIR /lambdabuild
 COPY clean_lambda_directory_before_zip_python%(two_or_three)s.py /
 COPY sourcecode/ /lambdabuild/
+RUN find /lambdabuild/ -name '__pycache__' | xargs rm -rf
 RUN python%(three)s /clean_lambda_directory_before_zip_python%(two_or_three)s.py . %(entry_points)s
 COPY rawfiles/ /lambdabuild/
 RUN find -name '*.so*' | xargs -I @ strip @

lambdabuild/runtimeinfo.py

@@ -19,9 +19,26 @@
         two_or_three="3",
         three="3",
     ),
+    '3.9': dict(
+        base_image="public.ecr.aws/sam/build-python3.9",
+        two_or_three="3",
+        three="3",
+    ),
+    '3.11': dict(
+        base_image="public.ecr.aws/sam/build-python3.11",
+        two_or_three="3",
+        three="3",
+    ),
+    '3.13': dict(
+        base_image="public.ecr.aws/sam/build-python3.13",
+        two_or_three="3",
+        three="3",
+    ),
 }
 
 BASE_IMAGES = [
+    "public.ecr.aws/sam/build-python3.13",
+    "public.ecr.aws/sam/build-python3.9",
     "lambci/lambda:build-python3.8",
     "lambci/lambda:build-python3.7",
     "lambci/lambda:build-python3.6",

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="lambdabuild",
-    version="0.2.5",
+    version="0.2.11",
     author="Shlomi Matichin",
     author_email="shlomomatichin@gmail.com",
     description="Easily build AWS lambda artifacts and layers, correctly",