Lists/.github/workflows/normalize-sync.yml at main · blocklistproject/Lists · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
name: Normalize and Sync Formats

on:
  workflow_dispatch:
    inputs:
      commit_changes:
        description: 'Commit and push changes'
        required: false
        type: boolean
        default: true
      dry_run:
        description: 'Dry run (show changes without committing)'
        required: false
        type: boolean
        default: false

permissions:
  contents: write
  pull-requests: write

jobs:
  normalize-and-sync:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 1

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

      - name: Run normalize and sync script
        id: normalize
        run: |
          echo "Starting normalization and sync..."
          python3 scripts/normalize_and_sync.py 2>&1 | tee normalize_output.txt

          # Capture exit code
          EXIT_CODE=${PIPESTATUS[0]}

          # Check if there are changes
          if git diff --quiet; then
            echo "has_changes=false" >> $GITHUB_OUTPUT
            echo "No changes detected"
          else
            echo "has_changes=true" >> $GITHUB_OUTPUT
            echo "Changes detected"
          fi

          exit $EXIT_CODE

      - name: Show changes
        if: steps.normalize.outputs.has_changes == 'true'
        run: |
          echo "=== Changes Summary ==="
          git status
          echo ""
          echo "=== Modified Files ==="
          git diff --stat

      - name: Commit and push changes
        if: |
          steps.normalize.outputs.has_changes == 'true' &&
          inputs.dry_run == false &&
          inputs.commit_changes == true
        run: |
          git config --local user.email "github-actions[bot]@users.noreply.github.com"
          git config --local user.name "github-actions[bot]"

          # Get list of modified files for commit message
          MODIFIED_LISTS=$(git diff --name-only | grep '\.txt$' | grep -v '/' | sed 's/\.txt$//' | sort | tr '\n' ', ' | sed 's/,$//')

          if [ -n "$MODIFIED_LISTS" ]; then
            git commit -m "chore: normalize and sync formats" \
                       -m "Cleaned invalid entries and regenerated formats for: ${MODIFIED_LISTS}" \
                       -m "- Removed invalid domain entries (paths, fragments, non-ASCII)" \
                       -m "- Regenerated all format variations to ensure sync" \
                       -m "- Automated via normalize-and-sync workflow"
          else
            git commit -m "chore: normalize and sync formats" \
                       -m "Regenerated format files to ensure synchronization." \
                       -m "- Automated via normalize-and-sync workflow"
          fi

          git add -A
          git push

      - name: Create summary
        if: always()
        run: |
          echo "## Normalize and Sync Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          if [ -f normalize_output.txt ]; then
            echo "### Script Output" >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY
            cat normalize_output.txt >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
          fi

          if [ "${{ steps.normalize.outputs.has_changes }}" == "true" ]; then
            echo "### Changes Detected" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "**Modified Files:**" >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY
            git diff --stat >> $GITHUB_STEP_SUMMARY
            echo '```' >> $GITHUB_STEP_SUMMARY

            if [ "${{ inputs.dry_run }}" == "true" ]; then
              echo "" >> $GITHUB_STEP_SUMMARY
              echo "⚠️ **Dry run mode** - Changes were not committed" >> $GITHUB_STEP_SUMMARY
            elif [ "${{ inputs.commit_changes }}" == "false" ]; then
              echo "" >> $GITHUB_STEP_SUMMARY
              echo "ℹ️ Changes were not committed (commit_changes=false)" >> $GITHUB_STEP_SUMMARY
            else
              echo "" >> $GITHUB_STEP_SUMMARY
              echo "✅ Changes were committed and pushed" >> $GITHUB_STEP_SUMMARY
            fi
          else
            echo "✅ No changes needed - all lists are already clean and in sync" >> $GITHUB_STEP_SUMMARY
          fi