create smithy plugin to generate BDD ruleset

Author: sbiscigl

generated/src/aws-cpp-sdk-s3/source/S3EndpointRules.cpp

@@ -18,7 +18,6 @@
 #include <aws/s3/S3Request.h>
 #include <aws/s3/S3ServiceClientModel.h>
 #include <aws/s3/S3_EXPORTS.h>
-#include <aws/s3/internal/S3EndpointRules.h>
 #include <aws/s3/model/AbacStatus.h>
 #include <aws/s3/model/AbortIncompleteMultipartUpload.h>
 #include <aws/s3/model/AbortMultipartUploadRequest.h>

generated/tests/s3-gen-tests/S3IncludeTests.cpp

@@ -45,6 +45,9 @@ tasks.register("generate-smithy-build") {
         val c2jMapStr: String = project.findProperty("c2jMap")?.toString() ?: "{}"
         val namespaceMappings: String = project.findProperty("namespaceMappings")?.toString() ?: "{}"
         val generateModels: Boolean = project.findProperty("generateModels")?.toString()?.toBoolean() ?: false
+        val generateEndpointRules: Boolean = project.findProperty("generateEndpointRules")?.toString()?.toBoolean() ?: false
+        val bddBytecoderPath: String = project.findProperty("bddBytecoderPath")?.toString() ?: ""
+        val pythonExecutable: String = project.findProperty("pythonExecutable")?.toString() ?: "python3"
 
         fileTree(models).filter { it.isFile }.files.forEach eachFile@{ file ->
             val model = Model.assembler()
@@ -74,6 +77,14 @@ tasks.register("generate-smithy-build") {
                     .withMember("namespaceMappings", Node.from(namespaceMappings))
                     .build())
             }
+            if (generateEndpointRules) {
+                pluginsNode = pluginsNode.withMember("smithy-cpp-codegen-endpoint-rules", Node.objectNodeBuilder()
+                    .withMember("c2jMap", Node.from(c2jMapStr))
+                    .withMember("namespaceMappings", Node.from(namespaceMappings))
+                    .withMember("bddBytecoderPath", Node.from(bddBytecoderPath))
+                    .withMember("pythonExecutable", Node.from(pythonExecutable))
+                    .build())
+            }
 
             val projectionContents = Node.objectNodeBuilder()
                 .withMember("imports", Node.fromStrings("${models.absolutePath}${File.separator}${file.name}"))
@@ -127,6 +138,14 @@ tasks.register("generate-smithy-build") {
                         .withMember("namespaceMappings", Node.from(namespaceMappings))
                         .build())
                 }
+                if (generateEndpointRules) {
+                    s3CrtPluginsNode = s3CrtPluginsNode.withMember("smithy-cpp-codegen-endpoint-rules", Node.objectNodeBuilder()
+                        .withMember("c2jMap", Node.from(c2jMapStr))
+                        .withMember("namespaceMappings", Node.from(namespaceMappings))
+                        .withMember("bddBytecoderPath", Node.from(bddBytecoderPath))
+                        .withMember("pythonExecutable", Node.from(pythonExecutable))
+                        .build())
+                }
                 val s3CrtProjectionContents = Node.objectNodeBuilder()
                     .withMember("imports", Node.fromStrings(s3ModelFile.absolutePath))
                     .withMember("plugins", s3CrtPluginsNode)

tools/code-generation/smithy/cpp-codegen/build.gradle.kts

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=file:///opt/gradle-install.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.6.1-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

tools/code-generation/smithy/cpp-codegen/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,69 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+package com.amazonaws.util.awsclientsmithygenerator.generators.endpointrules;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Shells out to the in-tree {@code bdd-bytecoder.py} compiler to turn the JSON form of a
+ * {@code smithy.rules#endpointBdd} trait into the binary BDD bytecode blob consumed by the
+ * CRT {@code BddEngine}. The bytecoder path and python executable are supplied by the caller
+ * (threaded in from {@code smithy_cpp_gen.py} as gradle properties) because the gradle working
+ * directory cannot reach the compiler under {@code crt/} on its own.
+ */
+public final class BddBytecoder {
+
+    private BddBytecoder() {}
+
+    /**
+     * @param pythonExecutable python interpreter to run (e.g. "python3").
+     * @param bytecoderPath    absolute path to bdd-bytecoder.py.
+     * @param traitJson        the endpointBdd trait serialized as JSON (the compiler's input).
+     * @param serviceLabel     used only to label temp files / error messages.
+     * @return the compiled binary bytecode.
+     */
+    public static byte[] compile(String pythonExecutable, String bytecoderPath, String traitJson, String serviceLabel) {
+        String tempPrefix = "bdd-" + serviceLabel.replaceAll("[^a-zA-Z0-9._-]", "_") + "-";
+        try (TempFile input = new TempFile(tempPrefix, ".json");
+             TempFile output = new TempFile(tempPrefix, ".bin")) {
+            Files.writeString(input.path, traitJson, StandardCharsets.UTF_8);
+
+            Process process = new ProcessBuilder(
+                    pythonExecutable, bytecoderPath, input.path.toString(), output.path.toString())
+                .redirectErrorStream(true)
+                .start();
+            String consoleOutput = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8);
+            int exitCode = process.waitFor();
+            if (exitCode != 0) {
+                throw new RuntimeException("bdd-bytecoder.py failed for '" + serviceLabel
+                    + "' (exit " + exitCode + "):\n" + consoleOutput);
+            }
+            byte[] bytecode = Files.readAllBytes(output.path);
+            if (bytecode.length == 0) {
+                throw new RuntimeException("bdd-bytecoder.py produced an empty blob for '" + serviceLabel + "'");
+            }
+            return bytecode;
+        } catch (IOException | InterruptedException e) {
+            throw new RuntimeException("Failed to run bdd-bytecoder.py for '" + serviceLabel + "'", e);
+        }
+    }
+
+    /** A temp file that deletes itself on close, so {@link #compile} can lean on try-with-resources. */
+    private static final class TempFile implements AutoCloseable {
+        private final Path path;
+
+        TempFile(String prefix, String suffix) throws IOException {
+            this.path = Files.createTempFile(prefix, suffix);
+        }
+
+        @Override
+        public void close() throws IOException {
+            Files.deleteIfExists(path);
+        }
+    }
+}

tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/endpointrules/BddBytecoder.java

@@ -0,0 +1,122 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+package com.amazonaws.util.awsclientsmithygenerator.generators.endpointrules;
+
+import com.amazonaws.util.awsclientsmithygenerator.generators.CppWriterDelegator;
+import com.amazonaws.util.awsclientsmithygenerator.generators.ServiceNameUtil;
+import software.amazon.smithy.build.PluginContext;
+import software.amazon.smithy.build.SmithyBuildPlugin;
+import software.amazon.smithy.model.Model;
+import software.amazon.smithy.model.node.Node;
+import software.amazon.smithy.model.node.ObjectNode;
+import software.amazon.smithy.model.node.StringNode;
+import software.amazon.smithy.model.shapes.ServiceShape;
+import software.amazon.smithy.model.shapes.ShapeId;
+
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Generates the {@code <Prefix>EndpointRules.{h,cpp}} pair carrying the compiled BDD bytecode blob.
+ *
+ * <p>This is the smithy-side counterpart to the C2J {@code --skip-endpoint-rules-blob} flag: when that
+ * flag is set the legacy generator deliberately emits nothing for these two files, leaving this plugin
+ * to write them whole. The emitted ABI is identical to the JSON path (see {@link EndpointRulesRenderer});
+ * only the blob contents change from ruleset JSON to binary BDD bytecode.
+ *
+ * <p>The BDD is read from the {@code smithy.rules#endpointBdd} trait on the service shape and compiled by
+ * shelling out to the in-tree {@code bdd-bytecoder.py}. The interpreter and compiler paths are supplied via
+ * the {@code pythonExecutable} and {@code bddBytecoderPath} plugin settings (threaded in from
+ * {@code smithy_cpp_gen.py}) because gradle's working directory cannot reach the compiler under {@code crt/}.
+ */
+public class EndpointRulesCodegenPlugin implements SmithyBuildPlugin {
+
+    private static final ShapeId ENDPOINT_BDD_TRAIT = ShapeId.from("smithy.rules#endpointBdd");
+    private static final String DEFAULT_PYTHON = "python3";
+
+    @Override
+    public String getName() {
+        return "smithy-cpp-codegen-endpoint-rules";
+    }
+
+    @Override
+    public void execute(PluginContext context) {
+        Model model = context.getModel();
+
+        // Mock projections stand in for legacy services with no Smithy model, hence no BDD to compile.
+        if (context.getProjectionName().endsWith(".mock")) {
+            return;
+        }
+
+        ObjectNode settings = context.getSettings();
+        Map<String, String> serviceMap = parseMapSetting(settings, "c2jMap");
+        Map<String, String> namespaceMap = parseNamespaceMap(settings);
+        String pythonExecutable = settings.getStringMemberOrDefault("pythonExecutable", DEFAULT_PYTHON);
+        String bddBytecoderPath = settings.getStringMember("bddBytecoderPath")
+            .map(n -> n.getValue())
+            .orElseThrow(() -> new IllegalStateException(
+                "endpoint-rules plugin requires the 'bddBytecoderPath' setting (path to bdd-bytecoder.py)"));
+
+        CppWriterDelegator writerDelegator = new CppWriterDelegator(context.getFileManifest());
+
+        // Services without an endpointBdd trait keep resolving via the JSON path, so skip them here.
+        model.getServiceShapes().stream()
+            .map(service -> ServiceNameUtil.processS3CrtProjection(service, context.getProjectionName()))
+            .filter(service -> service.hasTrait(ENDPOINT_BDD_TRAIT))
+            .forEach(service -> generateEndpointRules(
+                service, serviceMap, namespaceMap, pythonExecutable, bddBytecoderPath, writerDelegator));
+
+        writerDelegator.flushWriters();
+    }
+
+    private void generateEndpointRules(ServiceShape service, Map<String, String> serviceMap,
+                                       Map<String, String> namespaceMap, String pythonExecutable,
+                                       String bddBytecoderPath, CppWriterDelegator writerDelegator) {
+        String serviceName = ServiceNameUtil.getServiceName(service);
+        String smithyServiceName = ServiceNameUtil.getSmithyServiceName(service, serviceMap);
+        String exportMacro = ServiceNameUtil.getExportMacro(service, serviceMap);
+        String namespace = namespaceMap.getOrDefault(smithyServiceName, serviceName);
+
+        String traitJson = Node.printJson(service.findTrait(ENDPOINT_BDD_TRAIT).orElseThrow().toNode());
+        byte[] bytecode = BddBytecoder.compile(pythonExecutable, bddBytecoderPath, traitJson, smithyServiceName);
+
+        writerDelegator.useFileWriter(
+            "include/aws/" + smithyServiceName + "/internal/" + namespace + "EndpointRules.h",
+            writer -> EndpointRulesRenderer.renderHeader(writer, namespace, smithyServiceName, exportMacro));
+        writerDelegator.useFileWriter(
+            "source/" + namespace + "EndpointRules.cpp",
+            writer -> EndpointRulesRenderer.renderSource(writer, namespace, smithyServiceName, bytecode));
+    }
+
+    private Map<String, String> parseMapSetting(ObjectNode settings, String key) {
+        return settings.getMember(key)
+                .filter(Node::isStringNode)
+                .map(Node::expectStringNode)
+                .map(StringNode::getValue)
+                .map(Node::parseJsonWithComments)
+                .map(Node::expectObjectNode)
+                .map(mapNode -> mapNode.getMembers().entrySet().stream()
+                        .collect(Collectors.toMap(
+                                entry -> entry.getKey().getValue(),
+                                entry -> entry.getValue().expectStringNode().getValue())))
+                .orElse(Map.of());
+    }
+
+    private Map<String, String> parseNamespaceMap(ObjectNode settings) {
+        return settings.getMember("namespaceMappings")
+            .map(node -> node.expectStringNode().getValue())
+            .map(jsonStr -> Node.parseJsonWithComments(jsonStr).expectObjectNode())
+            .map(node -> node.getMembers().entrySet().stream()
+                .collect(Collectors.toMap(
+                    entry -> entry.getKey().getValue(),
+                    entry -> sanitize(entry.getValue().expectStringNode().getValue()))))
+            .orElse(Map.of());
+    }
+
+    private static String sanitize(String s) {
+        return s.replace(" ", "").replace("-", "").replace("_", "")
+            .replace("Amazon", "").replace("AWS", "").replace("/", "");
+    }
+}

tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/endpointrules/EndpointRulesCodegenPlugin.java

@@ -0,0 +1,116 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+package com.amazonaws.util.awsclientsmithygenerator.generators.endpointrules;
+
+import com.amazonaws.util.awsclientsmithygenerator.generators.CppWriter;
+
+/**
+ * Renders the {@code <Prefix>EndpointRules} header and source that carry a service's compiled BDD
+ * bytecode blob. The emitted ABI matches the C2J-generated {@code EndpointRules} class exactly
+ * (class {@code AWS_<SVC>_LOCAL <Prefix>EndpointRules} exposing {@code RulesBlobStrLen},
+ * {@code RulesBlobSize} and {@code GetRulesBlob()}) so the core {@code CrtEndpointProvider} consumes
+ * it unchanged. The only difference from the JSON path is that the blob bytes are binary BDD
+ * bytecode instead of ruleset JSON.
+ */
+public final class EndpointRulesRenderer {
+
+    // Byte-array lines wrap after this many entries, mirroring the C2J template's 25-per-line layout.
+    // clang-format re-flows afterwards, so this only affects the pre-format footprint.
+    private static final int BYTES_PER_LINE = 25;
+
+    private EndpointRulesRenderer() {}
+
+    /**
+     * include/aws/&lt;svc&gt;/internal/&lt;Prefix&gt;EndpointRules.h
+     */
+    public static void renderHeader(CppWriter writer, String namespace, String smithyServiceName,
+                                    String exportMacro) {
+        String localMacro = toLocalMacro(exportMacro);
+        writer.write("/**");
+        writer.write(" * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.");
+        writer.write(" * SPDX-License-Identifier: Apache-2.0.");
+        writer.write(" */");
+        writer.write("");
+        writer.write("#pragma once");
+        writer.write("#include <aws/$1L/$2L_EXPORTS.h>", smithyServiceName, namespace);
+        writer.write("");
+        writer.write("#include <cstddef>");
+        writer.write("");
+        writer.write("namespace Aws {");
+        writer.write("namespace $L {", namespace);
+        writer.write("class $1L $2LEndpointRules {", localMacro, namespace);
+        writer.write(" public:");
+        writer.write("  static const size_t RulesBlobStrLen;");
+        writer.write("  static const size_t RulesBlobSize;");
+        writer.write("");
+        writer.write("  static const char* GetRulesBlob();");
+        writer.write("};");
+        writer.write("}  // namespace $L", namespace);
+        writer.write("}  // namespace Aws");
+    }
+
+    /**
+     * source/&lt;Prefix&gt;EndpointRules.cpp
+     */
+    public static void renderSource(CppWriter writer, String namespace, String smithyServiceName,
+                                    byte[] bytecode) {
+        writer.write("/**");
+        writer.write(" * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.");
+        writer.write(" * SPDX-License-Identifier: Apache-2.0.");
+        writer.write(" */");
+        writer.write("");
+        writer.write("#include <aws/core/utils/memory/stl/AWSArray.h>");
+        writer.write("#include <aws/$1L/internal/$2LEndpointRules.h>", smithyServiceName, namespace);
+        writer.write("");
+        writer.write("namespace Aws {");
+        writer.write("namespace $L {", namespace);
+        // Binary bytecode has no NUL terminator; StrLen and Size are the same for the BDD blob.
+        writer.write("const size_t $1LEndpointRules::RulesBlobStrLen = $2L;", namespace, Integer.toString(bytecode.length));
+        writer.write("const size_t $1LEndpointRules::RulesBlobSize = $2L;", namespace, Integer.toString(bytecode.length));
+        writer.write("");
+        writer.write("using RulesBlobT = Aws::Array<const char, $LEndpointRules::RulesBlobSize>;", namespace);
+        writer.write("static constexpr RulesBlobT RulesBlob = {");
+        writeByteArray(writer, bytecode);
+        writer.write("};");
+        writer.write("");
+        writer.write("const char* $LEndpointRules::GetRulesBlob() { return RulesBlob.data(); }", namespace);
+        writer.write("}  // namespace $L", namespace);
+        writer.write("}  // namespace Aws");
+    }
+
+    private static void writeByteArray(CppWriter writer, byte[] bytecode) {
+        StringBuilder line = new StringBuilder("    {");
+        for (int i = 0; i < bytecode.length; i++) {
+            line.append(charLiteral(bytecode[i]));
+            if (i != bytecode.length - 1) {
+                line.append(',');
+            }
+            if ((i + 1) % BYTES_PER_LINE == 0 && i != bytecode.length - 1) {
+                writer.write("$L", line.toString());
+                line.setLength(0);
+                line.append("     ");
+            }
+        }
+        line.append("}");
+        writer.write("$L", line.toString());
+    }
+
+    /**
+     * Emits a byte as a C++ char literal. Bytecode is arbitrary binary, so every byte is written as a
+     * hex escape ('\xNN'); this is unambiguous, avoids narrowing warnings on values &gt; 0x7F, and never
+     * needs the printable/escape special-casing the C2J JSON template relied on.
+     */
+    private static String charLiteral(byte b) {
+        return String.format("'\\x%02x'", b & 0xFF);
+    }
+
+    private static String toLocalMacro(String exportMacro) {
+        // exportMacro is "AWS_<SVC>_API"; the local (hidden-visibility) macro is "AWS_<SVC>_LOCAL".
+        if (exportMacro.endsWith("_API")) {
+            return exportMacro.substring(0, exportMacro.length() - "_API".length()) + "_LOCAL";
+        }
+        return exportMacro + "_LOCAL";
+    }
+}

tools/code-generation/smithy/cpp-codegen/smithy-cpp-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/endpointrules/EndpointRulesRenderer.java

@@ -1,3 +1,4 @@
 com.amazonaws.util.awsclientsmithygenerator.generators.pagination.PaginationCodegenPlugin
 com.amazonaws.util.awsclientsmithygenerator.generators.waiters.WaiterCodegenPlugin
 com.amazonaws.util.awsclientsmithygenerator.generators.model.ModelCodegenPlugin
+com.amazonaws.util.awsclientsmithygenerator.generators.endpointrules.EndpointRulesCodegenPlugin