Skip to content

Commit fb2d761

Browse files
committed
Improve CI sync-cac-oscal when rule/var description/options change
Signed-off-by: Sophia Wang <huiwang@redhat.com>
1 parent cd02e4e commit fb2d761

2 files changed

Lines changed: 172 additions & 8 deletions

File tree

.github/workflows/sync-cac-oscal.yml

Lines changed: 33 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,10 @@ jobs:
1717
python-version: '3.9'
1818
# Step 2: Install Git
1919
- name: Install Git
20-
run: sudo apt-get update && sudo apt-get install -y git
20+
run: |
21+
sudo apt-get update && sudo apt-get install -y git
22+
python -m pip install --upgrade pip
23+
pip install ruamel.yaml
2124
# Step 3: Checkout the CaC repo
2225
- name: Checkout CaC repo
2326
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
@@ -62,16 +65,38 @@ jobs:
6265
if: ${{ env.SKIP == 'false' }}
6366
id: changed-files
6467
run: |
65-
repo=${{ github.repository }}
68+
OWNER="ComplianceAsCode"
69+
REPO="content"
6670
# Fetch all pages of the files for the pull request
67-
url="repos/$repo/pulls/${{ env.PR_NUMBER }}/files"
71+
url="repos/$OWNER/$REPO/pulls/${{ env.PR_NUMBER }}/files"
6872
response=$(gh api "$url" --paginate)
6973
echo "$response" | jq -r '.[].filename' > filenames.txt
7074
echo "CHANGE_FOUND=false" >> $GITHUB_ENV
71-
if grep -E "controls/|.profile|rule.yml|.var" filenames.txt ; then
75+
cd cac-content
76+
while IFS= read -r line; do
77+
if echo "$line" | grep -qE "controls/|\.profile"; then
78+
echo "$line" >> valid_files.txt
79+
# Only the rule.description, var.description, and var.options updates
80+
# will trigger the sync.
81+
elif [[ "$line" == *"rule.yml" || "$line" == *".var" ]]; then
82+
python utils/compare_rule_var.py \
83+
--owner $OWNER \
84+
--repo $REPO \
85+
${{ env.PR_NUMBER }} \
86+
"$line" \
87+
description options \
88+
|| {
89+
echo "Change detected! Running follow-up actions..."
90+
echo "The description/options in the $line have been updated."
91+
echo "$line" >> valid_files.txt
92+
}
93+
fi
94+
done < ../filenames.txt
95+
if [[ -f valid_files.txt ]]; then
96+
echo "Shows valid_files:"
97+
cat valid_files.txt
7298
echo "CHANGE_FOUND=true" >> $GITHUB_ENV
7399
fi
74-
cat filenames.txt
75100
env:
76101
GH_TOKEN: ${{ steps.app-token.outputs.token }}
77102
# Step 6: Setup the complyscribe environment
@@ -130,8 +155,7 @@ jobs:
130155
- name: Handle the detected updates
131156
if: ${{ env.CHANGE_FOUND == 'true' }}
132157
run: |
133-
cat filenames.txt
134-
python cac-content/utils/handle_detected_updates.py 'filenames.txt' > updates 2>&1
158+
python cac-content/utils/handle_detected_updates.py 'cac-content/valid_files.txt' > updates 2>&1
135159
cd complyscribe && source venv/bin/activate
136160
RH_PRODUCTS=${{ env.RH_PRODUCTS }}
137161
i=0
@@ -162,6 +186,7 @@ jobs:
162186
done < "$GITHUB_WORKSPACE"/updates
163187
# Step 10: Check if there is any existing open PR
164188
- name: Check if there is any existing open PR
189+
if: ${{ env.CHANGE_FOUND == 'true' }}
165190
run: |
166191
cd oscal-content
167192
# Use the GitHub CLI to search for an open PR.
@@ -332,7 +357,7 @@ jobs:
332357
--head $BRANCH_NAME --state open --json id \
333358
| jq length)
334359
if [ "$PR_EXISTS" -gt 0 ]; then
335-
echo "PR ${{ env.BRANCH_NAME }} already exists. Skipping PR creation.
360+
echo "PR ${{ env.BRANCH_NAME }} already exists. Skipping PR creation."
336361
echo "Add a comment for the CAC PR ${{ env.PR_NUMBER }}."
337362
gh pr comment ${{ env.BRANCH_NAME }} --body "${PR_BODY}"
338363
else

utils/compare_rule_var.py

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
#!/usr/bin/env python3
2+
import sys
3+
import subprocess
4+
import json
5+
import base64
6+
import argparse
7+
import io
8+
from ruamel.yaml import YAML
9+
from typing import Optional
10+
11+
def run_command(command: list[str]) -> tuple[int, str, str]:
12+
"""Runs a shell command and returns its exit code, stdout, and stderr."""
13+
res = subprocess.run(command, capture_output=True, text=True, check=False)
14+
return res.returncode, res.stdout, res.stderr
15+
16+
def get_file_content_from_pr(owner: str, repo: str, sha: str, file_path: str) -> str:
17+
"""Fetches file content for a given commit SHA using the GitHub API.
18+
The base64 encoding and decoding step is necessary because of how the
19+
GitHub API is designed to transmit file content.
20+
"""
21+
api_path = f"/repos/{owner}/{repo}/contents/{file_path}?ref={sha}"
22+
print(f"-> Fetching file content for '{file_path}' from commit {sha[:7]}...")
23+
24+
returncode, stdout, stderr = run_command(["gh", "api", api_path])
25+
26+
if returncode != 0:
27+
print(f"Exception: Could not fetch file. It may be new or deleted. (stderr: {stderr.strip()})")
28+
return ""
29+
30+
try:
31+
content_json = json.loads(stdout)
32+
encoded_content = content_json.get('content', '')
33+
if not encoded_content:
34+
return ""
35+
36+
return base64.b64decode(encoded_content).decode('utf-8')
37+
except (json.JSONDecodeError, KeyError, TypeError) as e:
38+
print(f"Exception: Could not parse or decode content. Error: {e}", file=sys.stderr)
39+
return ""
40+
41+
def parse_yaml_and_get_keys(content: str, keys_to_find: list[str]) -> Optional[str]:
42+
"""
43+
Parses a YAML content string, cleans it, and finds the values of ALL
44+
matching keys from the provided list. Returns a canonical YAML string
45+
of the found keys and their values for comparison.
46+
"""
47+
yaml = YAML(typ='safe')
48+
49+
# The rule.yml has '{{{...}}}' that can't be parsed by YMAL.
50+
# Pre-process content to remove invalid lines.
51+
lines = content.splitlines()
52+
clean_lines = [line for line in lines if not line.strip().startswith('{{{')]
53+
cleaned_content = "\n".join(clean_lines)
54+
55+
if not cleaned_content.strip():
56+
return None
57+
58+
try:
59+
data = yaml.load(cleaned_content)
60+
if not isinstance(data, dict):
61+
return None
62+
except Exception as e:
63+
print(f"Exception: Could not parse YAML content. Error: {e}", file=sys.stderr)
64+
return None
65+
66+
# Create a dictionary of all found keys and their values
67+
found_data = {}
68+
for key in keys_to_find:
69+
if key in data:
70+
found_data[key] = data.get(key)
71+
72+
if not found_data:
73+
return None
74+
75+
# For consistent comparison, convert the found data dict back to a YAML string
76+
string_stream = io.StringIO()
77+
yaml.dump(found_data, string_stream)
78+
return string_stream.getvalue().strip()
79+
80+
def main():
81+
"""Main function to fetch PR files, parse them, and compare keys."""
82+
parser = argparse.ArgumentParser(
83+
description="Compare specific keys in a YAML file between a PR's base and head branches."
84+
)
85+
parser.add_argument("--owner", required=True, help="The owner of the repository.")
86+
parser.add_argument("--repo", required=True, help="The name of the repository.")
87+
parser.add_argument("pr_number", type=int, help="The Pull Request number.")
88+
parser.add_argument("file_path", type=str, help="The full path to the file within the repository.")
89+
parser.add_argument("keys", nargs='+', help="One or more keys to check in order (e.g., description options).")
90+
args = parser.parse_args()
91+
92+
print(f"--- Analyzing '{args.file_path}' in PR #{args.pr_number} for keys: {args.keys} ---")
93+
print(f"Repository: {args.owner}/{args.repo}")
94+
95+
96+
# 1. Get the base and head commit SHAs.
97+
gh_pr_command = [
98+
"gh", "pr", "view", str(args.pr_number), "--repo", f"{args.owner}/{args.repo}", "--json", "baseRefOid,headRefOid"
99+
]
100+
returncode, stdout, stderr = run_command(gh_pr_command)
101+
102+
if returncode != 0:
103+
print(f"\nException: Failed to get PR details. gh stderr: {stderr.strip()}", file=sys.stderr)
104+
sys.exit(0)
105+
try:
106+
shas = json.loads(stdout)
107+
base_sha = shas['baseRefOid']
108+
head_sha = shas['headRefOid']
109+
print(f"Base SHA (Before): {base_sha}")
110+
print(f"Head SHA (After): {head_sha}\n")
111+
except (json.JSONDecodeError, KeyError) as e:
112+
print(f"\nException: Could not parse commit SHAs. Error: {e}", file=sys.stderr)
113+
sys.exit(0)
114+
115+
# 2. Fetch file content for both commits.
116+
before_content = get_file_content_from_pr(args.owner, args.repo, base_sha, args.file_path)
117+
after_content = get_file_content_from_pr(args.owner, args.repo, head_sha, args.file_path)
118+
119+
# 3. Parse the content in memory to get the desired values.
120+
before_value = parse_yaml_and_get_keys(before_content, args.keys)
121+
after_value = parse_yaml_and_get_keys(after_content, args.keys)
122+
123+
# 4. Compare the results and print the final output.
124+
print("\n--- Comparison Result ---")
125+
print(f"Value(s) in base branch:\n---\n{before_value}\n---")
126+
print(f"Value(s) in PR branch:\n---\n{after_value}\n---")
127+
128+
if before_value == after_value:
129+
print("\nNo changes detected for the specified keys.")
130+
sys.exit(0)
131+
else:
132+
print("\nChange detected for one of the specified keys!")
133+
print("\nCHANGE_FOUND=true")
134+
sys.exit(1)
135+
136+
if __name__ == "__main__":
137+
main()
138+
139+

0 commit comments

Comments
 (0)