refactor(jobs): migrate pingcap/tidb to the job-folder layout

jenkins/jobs/pingcap/tidb/latest/ghpr_build/Jenkinsfile

@@ -0,0 +1,98 @@
+// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline
+// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984
+@Library('tipipeline') _
+
+final K8S_NAMESPACE = "jenkins-tidb"
+final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh'
+final GIT_FULL_REPO_NAME = 'pingcap/tidb'
+final BRANCH_ALIAS = 'latest'
+final POD_TEMPLATE_FILE = "jenkins/jobs/${GIT_FULL_REPO_NAME}/${BRANCH_ALIAS}/${JOB_BASE_NAME}/pod.yaml"
+final REFS = readJSON(text: params.JOB_SPEC).refs
+
+prow.setPRDescription(REFS)
+pipeline {
+    agent {
+        kubernetes {
+            namespace K8S_NAMESPACE
+            yaml pod_label.withCiLabels(POD_TEMPLATE_FILE, REFS)
+            retries 2
+            workspaceVolume genericEphemeralVolume(accessModes: 'ReadWriteOnce', requestsSize: '150Gi', storageClassName: 'ci-rwo')
+            defaultContainer 'golang'
+        }
+    }
+    options {
+        timeout(time: 150, unit: 'MINUTES')
+    }
+    stages {
+        stage('Checkout') {
+            steps {
+                dir(REFS.repo) {
+                    script {
+                        prow.checkoutRefsWithCacheLock(REFS, timeout = 5, credentialsId = GIT_CREDENTIALS_ID, withSubmodule = true)
+                    }
+                }
+            }
+        }
+        stage('Prepare bazel workspace') {
+            steps {
+                dir(REFS.repo) {
+                    script { bazel.prepareWorkspace() }
+                }
+            }
+        }
+        stage("Build tidb-server community edition"){
+            steps {
+                dir(REFS.repo) {
+                    sh "make bazel_build"
+                }
+            }
+            post {
+                failure {
+                    dir(REFS.repo) {
+                        archiveArtifacts(artifacts: 'importer.log,tidb-server-check.log', allowEmptyArchive: true)
+                    }
+                }
+            }
+        }
+        stage("Build tidb-server enterprise edition") {
+            steps {
+                dir(REFS.repo) {
+                    sh "make enterprise-prepare enterprise-server-build && ./bin/tidb-server -V"
+                }
+            }
+        }
+        stage("Test plugin") {
+            when { not { expression { REFS.base_ref ==~ /^feature[\/_].*/ || REFS.base_ref ==~ /^release-fts-[0-9]+$/ } } } // skip for feature and release-fts branches.
+            steps {
+                dir('enterprise-plugin') {
+                    cache(path: "./", includes: '**/*', key: "git/pingcap-inc/enterprise-plugin/rev-${REFS.pulls[0].sha}", restoreKeys: ['git/pingcap-inc/enterprise-plugin/rev-']) {
+                        retry(2) {
+                            script {
+                                component.checkout('git@github.com:pingcap-inc/enterprise-plugin.git', 'plugin', REFS.base_ref, REFS.pulls[0].title, GIT_CREDENTIALS_ID)
+                            }
+                        }
+                    }
+                }
+                sh label: 'Test plugins', script: """
+                    mkdir -p plugin-so
+
+                    # compile go plugin: audit
+                    pushd enterprise-plugin/audit && go mod tidy && popd
+                    pushd ${REFS.repo} && go run ./cmd/pluginpkg -pkg-dir ../enterprise-plugin/audit -out-dir ../plugin-so && popd
+
+                    # compile go plugin: whitelist
+                    pushd enterprise-plugin/whitelist && go mod tidy && popd
+                    pushd ${REFS.repo} && go run ./cmd/pluginpkg -pkg-dir ../enterprise-plugin/whitelist -out-dir ../plugin-so && popd
+
+                    # test them.
+                    make server -C ${REFS.repo}
+                    ./${REFS.repo}/bin/tidb-server -plugin-dir=./plugin-so -plugin-load=audit-1,whitelist-1 | tee ./loading-plugin.log &
+
+                    sleep 30
+                    ps aux | grep tidb-server
+                    killall -9 -r tidb-server
+                """
+            }
+        }
+    }
+}

jenkins/jobs/pingcap/tidb/latest/ghpr_build/dsl.groovy

@@ -0,0 +1,39 @@
+// REF: https://<your-jenkins-server>/plugin/job-dsl/api-viewer/index.html
+final fullRepo = 'pingcap/tidb'
+final branchAlias = 'latest' // For trunk and latest release branches.
+final jobName = 'ghpr_build'
+
+pipelineJob("${fullRepo}/${jobName}") {
+    description("Build tidb artifacts (${fullRepo}/${jobName})")
+    logRotator {
+        daysToKeep(30)
+    }
+    parameters {
+        // Ref: https://docs.prow.k8s.io/docs/jobs/#job-environment-variables
+        stringParam("BUILD_ID")
+        stringParam("PROW_JOB_ID")
+        stringParam("JOB_SPEC")
+    }
+
+    definition {
+        cpsScm {
+            lightweight(true)
+            scriptPath("jenkins/jobs/pingcap/tidb/latest/ghpr_build/Jenkinsfile")
+            scm {
+                git{
+                    remote {
+                        url('https://github.com/PingCAP-QE/ci.git')
+                    }
+                    branch('main')
+                    extensions {
+                        cloneOptions {
+                            depth(1)
+                            shallow(true)
+                            timeout(5)
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

jenkins/jobs/pingcap/tidb/latest/ghpr_build/pod.yaml

@@ -0,0 +1,74 @@
+apiVersion: v1
+kind: Pod
+spec:
+  securityContext:
+    fsGroup: 1000
+  containers:
+    - name: golang
+      image: "us-docker.pkg.dev/pingcap-testing-account/internal/test/wangweizhen/tidb_image:go12320241009"
+      securityContext:
+        privileged: true
+      tty: true
+      resources:
+        requests:
+          cpu: "4"
+          memory: 32Gi
+        limits:
+          cpu: "8"
+          memory: 64Gi
+      volumeMounts:
+        - mountPath: /tmp
+          name: tmp
+        - name: bazel-rc # migration: keep
+          mountPath: /data/
+          readOnly: true
+        - name: containerinfo # migration: keep
+          mountPath: /etc/containerinfo
+      lifecycle:
+        postStart:
+          exec:
+            command:
+              - /bin/sh
+              - /data/bazel-prepare-in-container.sh
+  volumes:
+    - name: tmp
+      ephemeral:
+        volumeClaimTemplate:
+          spec:
+            accessModes:
+              - ReadWriteOnce
+            resources:
+              requests:
+                storage: 100Gi
+            storageClassName: ci-rwo
+    - name: bazel-rc
+      configMap:
+        name: bazel
+    - name: containerinfo
+      downwardAPI:
+        items:
+          - path: cpu_limit
+            resourceFieldRef:
+              containerName: golang
+              resource: limits.cpu
+          - path: cpu_request
+            resourceFieldRef:
+              containerName: golang
+              resource: requests.cpu
+          - path: mem_limit
+            resourceFieldRef:
+              containerName: golang
+              resource: limits.memory
+          - path: mem_request
+            resourceFieldRef:
+              containerName: golang
+              resource: requests.memory
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: kubernetes.io/arch
+                operator: In
+                values:
+                  - amd64

jenkins/jobs/pingcap/tidb/latest/ghpr_check/Jenkinsfile

@@ -0,0 +1,74 @@
+// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline
+// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984
+// should triggerd for master and latest releases branches
+@Library('tipipeline') _
+
+final K8S_NAMESPACE = "jenkins-tidb"
+final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh'
+final GIT_FULL_REPO_NAME = 'pingcap/tidb'
+final POD_TEMPLATE_FILE = "jenkins/jobs/pingcap/tidb/latest/ghpr_check/pod.yaml"
+final REFS = readJSON(text: params.JOB_SPEC).refs
+
+prow.setPRDescription(REFS)
+pipeline {
+    agent {
+        kubernetes {
+            namespace K8S_NAMESPACE
+            yaml pod_label.withCiLabels(POD_TEMPLATE_FILE, REFS)
+            retries 2
+            workspaceVolume genericEphemeralVolume(accessModes: 'ReadWriteOnce', requestsSize: '150Gi', storageClassName: 'ci-rwo')
+            defaultContainer 'golang'
+        }
+    }
+    options {
+        timeout(time: 60, unit: 'MINUTES')
+        parallelsAlwaysFailFast()
+    }
+    stages {
+        stage('Checkout') {
+            steps {
+                dir(REFS.repo) {
+                    script {
+                        prow.checkoutRefsWithCacheLock(REFS, timeout = 5, credentialsId = GIT_CREDENTIALS_ID, withSubmodule = true)
+                    }
+                }
+            }
+        }
+        stage('Prepare bazel workspace') {
+            steps {
+                dir(REFS.repo) {
+                    script { bazel.prepareWorkspace() }
+                }
+            }
+        }
+        stage("Checks") {
+            environment { CODECOV_TOKEN = credentials('codecov-token-tidb') }
+            // !!! concurrent go builds will encounter conflicts probabilistically.
+            steps {
+                dir(REFS.repo) {
+                    sh script: 'make gogenerate check integrationtest'
+                }
+            }
+            post {
+                success {
+                    dir(REFS.repo) {
+                        script {
+                            prow.uploadCoverageToCodecov(REFS, 'integration', './coverage.dat')
+                        }
+                    }
+                }
+                unsuccessful {
+                    dir(REFS.repo) {
+                        sh label: "archive log", script: """
+                        logs_dir='test_logs'
+                        mkdir -p \${logs_dir}
+                        mv tests/integrationtest/integration-test.out \${logs_dir} || true
+                        tar -czvf \${logs_dir}.tar.gz \${logs_dir} || true
+                        """
+                        archiveArtifacts(artifacts: '*.tar.gz', allowEmptyArchive: true)
+                    }
+                }
+            }
+        }
+    }
+}

jenkins/jobs/pingcap/tidb/latest/ghpr_check/dsl.groovy

@@ -0,0 +1,34 @@
+// REF: https://<your-jenkins-server>/plugin/job-dsl/api-viewer/index.html
+// For trunk and latest release branches.
+pipelineJob('pingcap/tidb/ghpr_check') {
+    logRotator {
+        daysToKeep(30)
+    }
+    parameters {
+        stringParam("BUILD_ID")
+        stringParam("PROW_JOB_ID")
+        stringParam("JOB_SPEC", "", "Prow job spec struct data")
+    }
+
+    definition {
+        cpsScm {
+            lightweight(true)
+            scriptPath("jenkins/jobs/pingcap/tidb/latest/ghpr_check/Jenkinsfile")
+            scm {
+                git{
+                    remote {
+                        url('https://github.com/PingCAP-QE/ci.git')
+                    }
+                    branch('main')
+                    extensions {
+                        cloneOptions {
+                            depth(1)
+                            shallow(true)
+                            timeout(5)
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

jenkins/jobs/pingcap/tidb/latest/ghpr_check/pod.yaml

@@ -0,0 +1,74 @@
+apiVersion: v1
+kind: Pod
+spec:
+  securityContext:
+    fsGroup: 1000
+  containers:
+    - name: golang
+      image: "us-docker.pkg.dev/pingcap-testing-account/internal/test/wangweizhen/tidb_image:go12320241009"
+      securityContext:
+        privileged: true
+      tty: true
+      resources:
+        requests:
+          memory: 8Gi
+          cpu: "8"
+        limits:
+          memory: 32Gi
+          cpu: "8"
+      volumeMounts:
+        - mountPath: /tmp
+          name: tmp
+        - name: bazel-rc
+          mountPath: /data/
+          readOnly: true
+        - name: containerinfo
+          mountPath: /etc/containerinfo
+      lifecycle:
+        postStart:
+          exec:
+            command:
+              - /bin/sh
+              - /data/bazel-prepare-in-container.sh
+  volumes:
+    - name: tmp
+      ephemeral:
+        volumeClaimTemplate:
+          spec:
+            accessModes:
+              - ReadWriteOnce
+            resources:
+              requests:
+                storage: 100Gi
+            storageClassName: ci-rwo
+    - name: bazel-rc
+      configMap:
+        name: bazel
+    - name: containerinfo
+      downwardAPI:
+        items:
+          - path: cpu_limit
+            resourceFieldRef:
+              containerName: golang
+              resource: limits.cpu
+          - path: cpu_request
+            resourceFieldRef:
+              containerName: golang
+              resource: requests.cpu
+          - path: mem_limit
+            resourceFieldRef:
+              containerName: golang
+              resource: limits.memory
+          - path: mem_request
+            resourceFieldRef:
+              containerName: golang
+              resource: requests.memory
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: kubernetes.io/arch
+                operator: In
+                values:
+                  - amd64