diff --git a/.github/workflows/ephemeral-env.yml b/.github/workflows/ephemeral-env.yml index 91a3b54b7..4c4747f5e 100644 --- a/.github/workflows/ephemeral-env.yml +++ b/.github/workflows/ephemeral-env.yml @@ -1,145 +1,124 @@ name: Ephemeral env workflow -# Example manual trigger: gh workflow run ephemeral-env.yml --ref fix_ephemerals --field comment_body="/testenv up" --field issue_number=666 +# Example manual trigger: +# gh workflow run ephemeral-env.yml --ref fix_ephemerals --field label_name="testenv-up" --field issue_number=666 on: - issue_comment: - types: [created] + pull_request_target: + types: + - labeled workflow_dispatch: inputs: - comment_body: - description: 'Comment body to simulate /testenv command' + label_name: + description: 'Label name to simulate label-based /testenv trigger' required: true - default: '/testenv up' + default: 'testenv-up' issue_number: description: 'Issue or PR number' required: true jobs: - ephemeral-env-comment: + ephemeral-env-label: concurrency: - group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-comment + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-label cancel-in-progress: true - name: Evaluate ephemeral env comment trigger (/testenv) + name: Evaluate ephemeral env label trigger runs-on: ubuntu-24.04 permissions: pull-requests: write outputs: - slash-command: ${{ steps.eval-body.outputs.result }} + slash-command: ${{ steps.eval-label.outputs.result }} feature-flags: ${{ steps.eval-feature-flags.outputs.result }} + sha: ${{ steps.get-sha.outputs.sha }} env: DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} steps: - - name: Debug - run: | - echo "Comment on PR #${{ github.event.issue.number }} by ${{ github.event.issue.user.login }}, ${{ github.event.comment.author_association }}" + - name: Check for the "testenv-up" label + id: eval-label + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + LABEL_NAME="${{ github.event.inputs.label_name }}" + else + LABEL_NAME="${{ github.event.label.name }}" + fi - - name: Eval comment body for /testenv slash command - uses: actions/github-script@v7 - env: - COMMENT_BODY: ${{ github.event.inputs.comment_body || github.event.comment.body }} - id: eval-body - with: - result-encoding: string - script: | - const pattern = /^\/testenv (up|down)/; - const result = pattern.exec(process.env.COMMENT_BODY || ''); - return result === null ? 'noop' : result[1]; + echo "Evaluating label: $LABEL_NAME" - - name: Looking for feature flags - uses: actions/github-script@v7 - env: - COMMENT_BODY: ${{ github.event.inputs.comment_body || github.event.comment.body }} - id: eval-feature-flags - with: - script: | - const pattern = /FEATURE_(\w+)=(\w+)/g; - let results = []; - [...process.env.COMMENT_BODY.matchAll(pattern)].forEach(match => { - const config = { - name: `SUPERSET_FEATURE_${match[1]}`, - value: match[2], - }; - results.push(config); - }); - return results; + if [[ "$LABEL_NAME" == "testenv-up" ]]; then + echo "result=up" >> $GITHUB_OUTPUT + else + echo "result=noop" >> $GITHUB_OUTPUT + exit 1 + fi - - name: Limit to committers - if: > - steps.eval-body.outputs.result != 'noop' && - github.event_name == 'issue_comment' && - github.event.comment.author_association != 'MEMBER' && - github.event.comment.author_association != 'OWNER' - uses: actions/github-script@v7 - with: - github-token: ${{ github.token }} - script: | - const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.'; - github.rest.issues.createComment({ - issue_number: ${{ github.event.issue.number }}, - owner: context.repo.owner, - repo: context.repo.repo, - body: errMsg - }); - core.setFailed(errMsg); + - name: Get event SHA + id: get-sha + run: | + echo "sha=${{ github.event.pull_request.head.sha }}" >> $GITHUB_OUTPUT + + - name: Looking for feature flags in PR description + uses: actions/github-script@v7 + id: eval-feature-flags + with: + script: | + const description = context.payload.pull_request + ? context.payload.pull_request.body || '' + : context.payload.inputs.pr_description || ''; + + const pattern = /FEATURE_(\w+)=(\w+)/g; + let results = []; + [...description.matchAll(pattern)].forEach(match => { + const config = { + name: `SUPERSET_FEATURE_${match[1]}`, + value: match[2], + }; + results.push(config); + }); + + return results; + + - name: Reply with confirmation comment + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const action = '${{ steps.eval-label.outputs.result }}'; + const user = context.actor; + const runId = context.runId; + const workflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`; + + const issueNumber = context.payload.pull_request + ? context.payload.pull_request.number + : context.payload.inputs.issue_number; + + if (!issueNumber) { + throw new Error("Issue number is not available."); + } + + const body = `@${user} Processing your ephemeral environment request [here](${workflowUrl}). Action: **${action}**.`; - - name: Reply with confirmation comment - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const issueNumber = ${{ github.event.inputs.issue_number || github.event.issue.number }}; - const user = '${{ github.event.comment.user.login || github.actor }}'; - const action = '${{ steps.eval-body.outputs.result }}'; - const runId = context.runId; - const workflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`; - const body = `@${user} Processing your ephemeral environment request [here](${workflowUrl}).`; - if (action !== 'noop') { await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber, body, }); - } - else { - core.setFailed('No ephemeral environment action detected.'); - } ephemeral-docker-build: concurrency: - group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-build + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-build cancel-in-progress: true - needs: ephemeral-env-comment - if: needs.ephemeral-env-comment.outputs.slash-command == 'up' + needs: ephemeral-env-label + if: needs.ephemeral-env-label.outputs.slash-command == 'up' name: ephemeral-docker-build runs-on: ubuntu-24.04 steps: - - name: Get Info from comment - uses: actions/github-script@v7 - id: get-pr-info - with: - script: | - const request = { - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: ${{ github.event.inputs.issue_number || github.event.issue.number }}, - }; - core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`); - const pr = await github.rest.pulls.get(request); - return pr.data; - - - name: Debug - id: get-sha - run: | - echo "sha=${{ fromJSON(steps.get-pr-info.outputs.result).head.sha }}" >> $GITHUB_OUTPUT - - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} : ${{steps.get-sha.outputs.sha}} )" + - name: "Checkout ${{ github.ref }} ( ${{ needs.ephemeral-env-label.outputs.sha }} : ${{steps.get-sha.outputs.sha}} )" uses: actions/checkout@v4 with: - ref: ${{ steps.get-sha.outputs.sha }} + ref: ${{ needs.ephemeral-env-label.outputs.sha }} persist-credentials: false - name: Setup Docker Environment @@ -181,14 +160,14 @@ jobs: env: ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} ECR_REPOSITORY: superset-ci - IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci + IMAGE_TAG: apache/superset:${{ needs.ephemeral-env-label.outputs.sha }}-ci run: | docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci docker push -a $ECR_REGISTRY/$ECR_REPOSITORY ephemeral-env-up: - needs: [ephemeral-env-comment, ephemeral-docker-build] - if: needs.ephemeral-env-comment.outputs.slash-command == 'up' + needs: [ephemeral-env-label, ephemeral-docker-build] + if: needs.ephemeral-env-label.outputs.slash-command == 'up' name: Spin up an ephemeral environment runs-on: ubuntu-24.04 permissions: @@ -196,120 +175,120 @@ jobs: pull-requests: write steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false + - uses: actions/checkout@v4 + with: + persist-credentials: false - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-west-2 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 - - name: Check target image exists in ECR - id: check-image - continue-on-error: true - run: | - aws ecr describe-images \ - --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \ - --repository-name superset-ci \ - --image-ids imageTag=pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci + - name: Check target image exists in ECR + id: check-image + continue-on-error: true + run: | + aws ecr describe-images \ + --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \ + --repository-name superset-ci \ + --image-ids imageTag=pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci - - name: Fail on missing container image - if: steps.check-image.outcome == 'failure' - uses: actions/github-script@v7 - with: - github-token: ${{ github.token }} - script: | - const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.'; - github.rest.issues.createComment({ - issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }}, - owner: context.repo.owner, - repo: context.repo.repo, - body: errMsg - }); - core.setFailed(errMsg); + - name: Fail on missing container image + if: steps.check-image.outcome == 'failure' + uses: actions/github-script@v7 + with: + github-token: ${{ github.token }} + script: | + const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.'; + github.rest.issues.createComment({ + issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: errMsg + }); + core.setFailed(errMsg); - - name: Fill in the new image ID in the Amazon ECS task definition - id: task-def - uses: aws-actions/amazon-ecs-render-task-definition@v1 - with: - task-definition: .github/workflows/ecs-task-definition.json - container-name: superset-ci - image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci + - name: Fill in the new image ID in the Amazon ECS task definition + id: task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: .github/workflows/ecs-task-definition.json + container-name: superset-ci + image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci - - name: Update env vars in the Amazon ECS task definition - run: | - cat <<< "$(jq '.containerDefinitions[0].environment += ${{ needs.ephemeral-env-comment.outputs.feature-flags }}' < ${{ steps.task-def.outputs.task-definition }})" > ${{ steps.task-def.outputs.task-definition }} + - name: Update env vars in the Amazon ECS task definition + run: | + cat <<< "$(jq '.containerDefinitions[0].environment += ${{ needs.ephemeral-env-label.outputs.feature-flags }}' < ${{ steps.task-def.outputs.task-definition }})" > ${{ steps.task-def.outputs.task-definition }} - - name: Describe ECS service - id: describe-services - run: | - echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT - - name: Create ECS service - id: create-service - if: steps.describe-services.outputs.active != 'true' - env: - ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974 - ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91 - run: | - aws ecs create-service \ - --cluster superset-ci \ - --service-name pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service \ - --task-definition superset-ci \ - --launch-type FARGATE \ - --desired-count 1 \ - --platform-version LATEST \ - --network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \ - --tags key=pr,value=${{ github.event.inputs.issue_number || github.event.issue.number }} key=github_user,value=${{ github.actor }} - - name: Deploy Amazon ECS task definition - id: deploy-task - uses: aws-actions/amazon-ecs-deploy-task-definition@v2 - with: - task-definition: ${{ steps.task-def.outputs.task-definition }} - service: pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service - cluster: superset-ci - wait-for-service-stability: true - wait-for-minutes: 10 + - name: Describe ECS service + id: describe-services + run: | + echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT + - name: Create ECS service + id: create-service + if: steps.describe-services.outputs.active != 'true' + env: + ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974 + ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91 + run: | + aws ecs create-service \ + --cluster superset-ci \ + --service-name pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service \ + --task-definition superset-ci \ + --launch-type FARGATE \ + --desired-count 1 \ + --platform-version LATEST \ + --network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \ + --tags key=pr,value=${{ github.event.inputs.issue_number || github.event.issue.number }} key=github_user,value=${{ github.actor }} + - name: Deploy Amazon ECS task definition + id: deploy-task + uses: aws-actions/amazon-ecs-deploy-task-definition@v2 + with: + task-definition: ${{ steps.task-def.outputs.task-definition }} + service: pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service + cluster: superset-ci + wait-for-service-stability: true + wait-for-minutes: 10 - - name: List tasks - id: list-tasks - run: | - echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT - - name: Get network interface - id: get-eni - run: | - echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name==\"networkInterfaceId\")) | .[0] | .value')" >> $GITHUB_OUTPUT - - name: Get public IP - id: get-ip - run: | - echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT - - name: Comment (success) - if: ${{ success() }} - uses: actions/github-script@v7 - with: - github-token: ${{github.token}} - script: | - github.rest.issues.createComment({ - issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }}, - owner: context.repo.owner, - repo: context.repo.repo, - body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.' - }) - - name: Comment (failure) - if: ${{ failure() }} - uses: actions/github-script@v7 - with: - github-token: ${{github.token}} - script: | - github.rest.issues.createComment({ - issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }}, - owner: context.repo.owner, - repo: context.repo.repo, - body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.' - }) + - name: List tasks + id: list-tasks + run: | + echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT + - name: Get network interface + id: get-eni + run: | + echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks[0].attachments[0].details | map(select(.name==\"networkInterfaceId\"))[0].value')" >> $GITHUB_OUTPUT + - name: Get public IP + id: get-ip + run: | + echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT + - name: Comment (success) + if: ${{ success() }} + uses: actions/github-script@v7 + with: + github-token: ${{github.token}} + script: | + github.rest.issues.createComment({ + issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: '@${{ github.actor }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.' + }) + - name: Comment (failure) + if: ${{ failure() }} + uses: actions/github-script@v7 + with: + github-token: ${{github.token}} + script: | + github.rest.issues.createComment({ + issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.' + })