Add opflex info

.github/scripts/extract_opflex_metadata.py

@@ -0,0 +1,102 @@
+import os
+import sys
+import yaml
+import json
+import subprocess
+
+if len(sys.argv) < 3:
+    print("Usage: python extract_opflex_metadata.py <repo_name> <output_file>")
+    sys.exit(1)
+
+repo_name = sys.argv[1]
+output_filepath = sys.argv[2]
+release_filepath = os.path.join(repo_name, "docs/release_artifacts/releases.yaml")
+
+def load_releases(filepath):
+    if not os.path.exists(filepath):
+        print(f"Error: {filepath} not found!")
+        return {}
+
+    with open(filepath, "r") as file:
+        try:
+            return yaml.safe_load(file) or {}
+        except yaml.YAMLError as exc:
+            print(f"Error parsing YAML: {exc}")
+            return {}
+
+def get_base_image_digest(image_name):
+    if image_name == "placeholder-image":
+        return None
+    try:
+        result = subprocess.check_output(
+            f"skopeo inspect --no-creds docker://{image_name} | jq -r '.Digest'",
+            shell=True,
+            text=True
+        ).strip()
+        return result.replace("sha256:", "")
+    except subprocess.CalledProcessError as e:
+        print(f"Error fetching digest for {image_name}: {e}")
+        return None
+
+def extract_opflex_metadata(releases):
+    release_list = []
+
+    for release in releases.get("releases", []):
+        release_tag = release.get("release_tag", "")
+        release_status = None
+        z_stream = None
+
+        for stream in release.get("release_streams", []):
+            release_name = stream.get("release_name", "")
+            if ".rc" in release_name:
+                continue
+            if release_name == release_tag:
+                release_status = stream.get("released", False)
+            elif release_name.endswith(".z"):
+                z_stream = stream
+
+        if z_stream:
+            if release_status is False:
+                print(f"Processing .z stream for {release_tag}")
+                for image in z_stream.get("container_images", []):
+                    if image.get("name") == "opflex":
+                        base_image = image.get("opflex-metadata", {}).get("base-image", "placeholder-image")
+                        stored_digest = image.get("opflex-metadata", {}).get("base-image-sha", "unknown")
+
+                        if base_image == "placeholder-image":
+                            print(f"Skipping digest lookup for {release_tag} (Base-Image is placeholder).")
+                            release_list.append({
+                                "release_tag": release_tag,
+                                "base_tag": f"{release_tag}-opflex-build-base",
+                                "base_image": base_image,
+                                "stored_digest": stored_digest,
+                                "latest_digest": "unknown",
+                                "update_digest": "true"
+                            })
+                            continue
+
+                        print(f"digest lookup for {release_tag}, {base_image}")
+                        latest_digest = get_base_image_digest(base_image)
+                        if latest_digest and latest_digest != stored_digest:
+                            print(f"Digest changed for {release_tag}: {stored_digest} → {latest_digest}")
+                            image["opflex-metadata"]["base-image-sha"] = latest_digest
+                            release_list.append({
+                                "release_tag": release_tag,
+                                "base_tag": f"{release_tag}-opflex-build-base",
+                                "base_image": base_image,
+                                "stored_digest": stored_digest,
+                                "latest_digest": latest_digest,
+                                "update_digest": "false"
+                            })
+                        else:
+                            print(f"Digest not changed for {release_tag}: stored:{stored_digest} :: latest:{latest_digest}")
+
+    return release_list
+
+releases = load_releases(release_filepath)
+release_list = extract_opflex_metadata(releases)
+
+with open(output_filepath, "w") as json_file:
+    json.dump(release_list, json_file, indent=2)
+
+print(f"Extracted OpFlex metadata saved to {output_filepath}")

.github/scripts/push_opflex_tags.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+# Ensure two arguments are provided
+if [ -z "$1" ] || [ -z "$2" ]; then
+  echo "Usage: $0 <metadata_json_file> <updated_metadata_json_file>"
+  exit 1
+fi
+
+METADATA_FILE=$1
+UPDATED_METADATA=$2
+CLONE_DIR="opflex"
+
+if [ ! -f "$METADATA_FILE" ]; then
+  echo "Error: File $METADATA_FILE not found!"
+  exit 1
+fi
+
+if [ ! -d "$CLONE_DIR" ]; then
+  echo "Error: OpFlex repository not found! Ensure it is cloned in the workspace."
+  exit 1
+fi
+
+jq -n '[]' > "$UPDATED_METADATA"
+
+# Process metadata
+jq -c '.[]' "$METADATA_FILE" | while read -r item; do
+  RELEASE_TAG=$(echo "$item" | jq -r '.release_tag')
+  BASE_TAG=$(echo "$item" | jq -r '.base_tag')
+  BASE_IMAGE=$(echo "$item" | jq -r '.base_image')
+  LATEST_DIGEST=$(echo "$item" | jq -r '.latest_digest')
+  UPDATE_DIGEST=$(echo "$item" | jq -r '.update_digest')
+
+  if [ -z "$BASE_TAG" ]; then
+    echo "Base tag is empty for $RELEASE_TAG. Skipping..."
+    continue
+  fi
+
+  cd "$CLONE_DIR" || exit 1
+  git fetch --tags
+  if git rev-parse "refs/tags/$BASE_TAG" >/dev/null 2>&1; then
+    git checkout "tags/$BASE_TAG"
+  else
+    echo "Tag $BASE_TAG does not exist. Skipping update for $RELEASE_TAG."
+    cd .. || exit 1
+    continue
+  fi
+
+  if [ "$UPDATE_DIGEST" == "true" ]; then
+    echo "Finding actual base image for $RELEASE_TAG..."
+    BASE_IMAGE=$(grep -E '^FROM' "docker/travis/Dockerfile-opflex" | awk '{print $2}')
+
+    if [ -z "$BASE_IMAGE" ]; then
+      echo "Failed to determine base image for $RELEASE_TAG. Skipping..."
+      cd .. || exit 1
+      continue
+    fi
+
+    echo "Base image for $RELEASE_TAG resolved to: $BASE_IMAGE"
+
+    LATEST_DIGEST=$(skopeo inspect --no-creds "docker://$BASE_IMAGE" | jq -r '.Digest' | sed 's/sha256://')
+
+    if [ -z "$LATEST_DIGEST" ]; then
+      echo "Failed to fetch digest for base image $BASE_IMAGE. Skipping..."
+      cd .. || exit 1
+      continue
+    fi
+
+    echo "Resolved digest for $RELEASE_TAG: $LATEST_DIGEST"
+  fi
+
+  tmp_file=$(mktemp)
+  jq --arg release_tag "$RELEASE_TAG" \
+      --arg base_image "$BASE_IMAGE" \
+      --arg latest_digest "$LATEST_DIGEST" \
+      '. + [{"release_tag": $release_tag, "base_image": $base_image, "latest_digest": $latest_digest, "update_digest": "true"}]' \
+      "$UPDATED_METADATA" > "$tmp_file" && mv "$tmp_file" "$UPDATED_METADATA"
+
+  git config --local user.name "noiro-generic"
+  git config --local user.email "noiro-generic@github.com"
+
+  if ! git remote | grep -q noiro-generic; then
+    git remote add noiro-generic https://noiro-generic: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@github.com/noironetworks/opflex.git
+    #git remote add noiro-generic https://noiro-generic:${{ secrets.NOIRO101_GENERIC_PAT }}@github.com/noironetworks/opflex.git
+  fi
+
+  TAG_MESSAGE="ACI Release $BASE_TAG Created by Github workflow ${GITHUB_WORKFLOW} ${GITHUB_SERVER_URL}/noironetworks/opflex/actions/runs/${GITHUB_RUN_ID}"
+  echo "$BASE_TAG: $TAG_MESSAGE"
+  # git tag -f -a "$BASE_TAG" -m "$TAG_MESSAGE"
+  # git push noiro-generic -f "$BASE_TAG"
+
+  cd .. || exit 1
+done

.github/scripts/update_releases.py

@@ -0,0 +1,93 @@
+import os
+import sys
+import yaml
+import json
+
+if len(sys.argv) < 3:
+    print("Usage: python update_releases_yaml.py <updated_metadata_file> <releases_file>")
+    sys.exit(1)
+
+updated_metadata_file = sys.argv[1]
+releases_file = sys.argv[2]
+
+def load_json(filepath):
+    if not os.path.exists(filepath):
+        print(f"Error: {filepath} not found!")
+        return []
+
+    with open(filepath, "r") as file:
+        try:
+            return json.load(file)
+        except json.JSONDecodeError as exc:
+            print(f"Error parsing JSON: {exc}")
+            return []
+
+def load_yaml(filepath):
+    if not os.path.exists(filepath):
+        print(f"Error: {filepath} not found!")
+        return {}
+
+    with open(filepath, "r") as file:
+        try:
+            return yaml.safe_load(file) or {}
+        except yaml.YAMLError as exc:
+            print(f"Error parsing YAML: {exc}")
+            return {}
+
+def save_yaml(filepath, data):
+    with open(filepath, "w") as file:
+        yaml.dump(data, file, default_flow_style=False)
+
+def update_releases(releases, updated_metadata):
+    updated = False
+    release_map = {item["release_tag"]: item for item in updated_metadata}
+
+    for release in releases.get("releases", []):
+        release_tag = release.get("release_tag", "")
+        release_status = None
+        z_stream = None
+
+        for stream in release.get("release_streams", []):
+            release_name = stream.get("release_name", "")
+            if ".rc" in release_name:
+                continue
+            if release_name == release_tag:
+                release_status = stream.get("released", False)
+            elif release_name.endswith(".z"):
+                z_stream = stream
+
+        if z_stream:
+            if release_status is False and release_tag in release_map:
+                print(f"Updating .z stream metadata for {release_tag}")
+
+                for image in z_stream.get("container_images", []):
+                    if image.get("name") == "opflex":
+                        opflex_metadata = release_map[release_tag]
+
+                        if "opflex-metadata" not in image:
+                            print(f"'opflex-metadata' missing for {release_tag}, initializing...")
+                            image["opflex-metadata"] = {
+                                "base-image": opflex_metadata["base_image"],
+                                "base-image-sha": opflex_metadata["latest_digest"]
+                            }
+                        else:
+                            print(f"Updating existing OpFlex metadata for {release_tag}")
+                            image["opflex-metadata"]["base-image"] = opflex_metadata["base_image"]
+                            image["opflex-metadata"]["base-image-sha"] = opflex_metadata["latest_digest"]
+
+                        updated = True
+
+    return updated
+
+updated_metadata = load_json(updated_metadata_file)
+releases = load_yaml(releases_file)
+
+if not updated_metadata or not releases:
+    print("No valid data to update. Exiting.")
+    sys.exit(1)
+
+if update_releases(releases, updated_metadata):
+    save_yaml(releases_file, releases)
+    print(f"Updated releases.yaml with new OpFlex metadata.")
+else:
+    print("No updates made to releases.yaml.")

.github/workflows/push-opflex-base-tag.yml

@@ -0,0 +1,89 @@
+name: Push Opflex Base Tags
+on:
+  schedule:
+    - cron: '0 0 * * *'  # Runs daily at midnight UTC
+  workflow_dispatch:
+  push: #for testing only, remove before merge
+    branches:
+      - 'add-opflex-info'
+jobs:
+  update-releases:
+    runs-on: ubuntu-latest
+    env:
+      CICD_REPO: "noironetworks/test-cicd-status"
+      OPFLEX_REPO: "noironetworks/opflex"
+      CICD_DIR: ${{ github.workspace }}/test-cicd-status
+      OPFLEX_DIR: ${{ github.workspace }}/opflex
+      WORKSPACE_DIR: ${{ github.workspace }}/workspace
+
+    steps:
+      - name: Checkout Main Repository
+        uses: actions/checkout@v4
+        # with:
+        #   token: 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 #${{ secrets.NOIRO101_GENERIC_PAT }}
+
+      - name: Checkout test-cicd-status Repository
+        uses: actions/checkout@v4
+        with:
+          token: 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 #${{ secrets.NOIRO101_GENERIC_PAT }}
+          repository: ${{ env.CICD_REPO }}
+          path: ${{ env.CICD_DIR }}
+          fetch-depth: 1
+
+      - name: Checkout OpFlex Repository
+        uses: actions/checkout@v4
+        with:
+          token: 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 #${{ secrets.NOIRO101_GENERIC_PAT }}
+          repository: ${{ env.OPFLEX_REPO }}
+          path: ${{ env.OPFLEX_DIR }}
+          fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+
+      - name: Install dependencies
+        run: pip install pyyaml jq
+
+      - name: Create workspace directory
+        run: mkdir -p ${{ env.WORKSPACE_DIR }}
+
+      - name: Extract current tags opflex metadata
+        run: python .github/scripts/extract_opflex_metadata.py ${{ env.CICD_DIR }} ${{ env.WORKSPACE_DIR }}/opflex_metadata.json
+
+      - name: Print Metadata File
+        run: |
+          echo "Printing opflex_metadata.json:"
+          cat "${{ env.WORKSPACE_DIR }}/opflex_metadata.json" || echo "ERR: Unable to read $METADATA_FILE"
+
+      - name: Push subsequent tags (only if digest changed)
+        run: bash .github/scripts/push_opflex_tags.sh ${{ env.WORKSPACE_DIR }}/opflex_metadata.json ${{ env.WORKSPACE_DIR }}/updated_metadata.json
+
+      - name: Print Updated Metadata File
+        run: |
+          echo "Printing updated_metadata.json:"
+          cat "${{ env.WORKSPACE_DIR }}/updated_metadata.json" || echo "ERR: Unable to read $METADATA_FILE"
+
+      - name: Update releases.yaml with new OpFlex metadata
+        run: python .github/scripts/update_releases.py ${{ env.WORKSPACE_DIR }}/updated_metadata.json ${{ env.CICD_DIR }}/docs/release_artifacts/releases.yaml
+
+      - name: Commit and Push Updated releases.yaml
+        run: |
+          cd ${{ env.CICD_DIR }}
+          git config --local user.name "noiro-generic"
+          git config --local user.email "noiro-generic@github.com"
+
+          if git diff --quiet docs/release_artifacts/releases.yaml; then
+            echo "No changes in releases.yaml. Skipping commit."
+          else
+            echo "Changes detected in releases.yaml. Committing..."
+
+            git add docs/release_artifacts/releases.yaml
+            git commit -m "Update stored digests for opflex base image in releases.yaml"
+            git remote set-url origin https://noiro-generic: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@github.com/noironetworks/test-cicd-status.git
+            # git remote set-url origin https://noiro-generic:${{ secrets.NOIRO101_GENERIC_PAT }}@github.com/noironetworks/test-cicd-status.git
+            git remote -v
+            git push origin main
+          fi
+

travis/globals.sh

@@ -15,4 +15,4 @@ RC_REGEX=${RC_PREFIX}[0-9]+
 QUAY_REGISTRY=quay.io/noirolabs
 QUAY_NOIRO_REGISTRY=quay.io/noiro
 DOCKER_REGISTRY=docker.io/noiro
-CICD_STATUS_REPO="https://github.com/noironetworks/cicd-status.git"
+CICD_STATUS_REPO="https://github.com/noironetworks/test-cicd-status.git"