330 lines
13 KiB
YAML
330 lines
13 KiB
YAML
name: Ephemeral env workflow
|
|
|
|
# Example manual trigger:
|
|
# gh workflow run ephemeral-env.yml --ref fix_ephemerals --field label_name="testenv-up" --field issue_number=666
|
|
|
|
on:
|
|
pull_request_target:
|
|
types:
|
|
- labeled
|
|
workflow_dispatch:
|
|
inputs:
|
|
label_name:
|
|
description: 'Label name to simulate label-based /testenv trigger'
|
|
required: true
|
|
default: 'testenv-up'
|
|
issue_number:
|
|
description: 'Issue or PR number'
|
|
required: true
|
|
|
|
jobs:
|
|
ephemeral-env-label:
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-label
|
|
cancel-in-progress: true
|
|
name: Evaluate ephemeral env label trigger
|
|
runs-on: ubuntu-24.04
|
|
permissions:
|
|
pull-requests: write
|
|
outputs:
|
|
slash-command: ${{ steps.eval-label.outputs.result }}
|
|
feature-flags: ${{ steps.eval-feature-flags.outputs.result }}
|
|
sha: ${{ steps.get-sha.outputs.sha }}
|
|
env:
|
|
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
|
|
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
|
|
steps:
|
|
- name: Check for the "testenv-up" label
|
|
id: eval-label
|
|
run: |
|
|
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
|
LABEL_NAME="${{ github.event.inputs.label_name }}"
|
|
else
|
|
LABEL_NAME="${{ github.event.label.name }}"
|
|
fi
|
|
|
|
echo "Evaluating label: $LABEL_NAME"
|
|
|
|
if [[ "$LABEL_NAME" == "testenv-up" ]]; then
|
|
echo "result=up" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "result=noop" >> $GITHUB_OUTPUT
|
|
fi
|
|
|
|
- name: Get event SHA
|
|
id: get-sha
|
|
if: steps.eval-label.outputs.result == 'up'
|
|
uses: actions/github-script@v7
|
|
with:
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
script: |
|
|
let prSha;
|
|
|
|
// If event is workflow_dispatch, use the issue_number from inputs
|
|
if (context.eventName === "workflow_dispatch") {
|
|
const prNumber = "${{ github.event.inputs.issue_number }}";
|
|
if (!prNumber) {
|
|
console.log("No PR number found.");
|
|
return;
|
|
}
|
|
|
|
// Fetch PR details using the provided issue_number
|
|
const { data: pr } = await github.rest.pulls.get({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
pull_number: prNumber
|
|
});
|
|
|
|
prSha = pr.head.sha;
|
|
} else {
|
|
// If it's not workflow_dispatch, use the PR head sha from the event
|
|
prSha = context.payload.pull_request.head.sha;
|
|
}
|
|
|
|
console.log(`PR SHA: ${prSha}`);
|
|
core.setOutput("sha", prSha);
|
|
|
|
- name: Looking for feature flags in PR description
|
|
uses: actions/github-script@v7
|
|
id: eval-feature-flags
|
|
if: steps.eval-label.outputs.result == 'up'
|
|
with:
|
|
script: |
|
|
const description = context.payload.pull_request
|
|
? context.payload.pull_request.body || ''
|
|
: context.payload.inputs.pr_description || '';
|
|
|
|
const pattern = /FEATURE_(\w+)=(\w+)/g;
|
|
let results = [];
|
|
[...description.matchAll(pattern)].forEach(match => {
|
|
const config = {
|
|
name: `SUPERSET_FEATURE_${match[1]}`,
|
|
value: match[2],
|
|
};
|
|
results.push(config);
|
|
});
|
|
|
|
return results;
|
|
|
|
- name: Reply with confirmation comment
|
|
uses: actions/github-script@v7
|
|
if: steps.eval-label.outputs.result == 'up'
|
|
with:
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
script: |
|
|
const action = '${{ steps.eval-label.outputs.result }}';
|
|
const user = context.actor;
|
|
const runId = context.runId;
|
|
const workflowUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${runId}`;
|
|
|
|
const issueNumber = context.payload.pull_request
|
|
? context.payload.pull_request.number
|
|
: context.payload.inputs.issue_number;
|
|
|
|
if (!issueNumber) {
|
|
throw new Error("Issue number is not available.");
|
|
}
|
|
|
|
const body = `@${user} Processing your ephemeral environment request [here](${workflowUrl}). Action: **${action}**.`;
|
|
|
|
await github.rest.issues.createComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
issue_number: issueNumber,
|
|
body,
|
|
});
|
|
|
|
ephemeral-docker-build:
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}-build
|
|
cancel-in-progress: true
|
|
needs: ephemeral-env-label
|
|
if: needs.ephemeral-env-label.outputs.slash-command == 'up'
|
|
name: ephemeral-docker-build
|
|
runs-on: ubuntu-24.04
|
|
steps:
|
|
- name: "Checkout ${{ github.ref }} ( ${{ needs.ephemeral-env-label.outputs.sha }} : ${{steps.get-sha.outputs.sha}} )"
|
|
uses: actions/checkout@v4
|
|
with:
|
|
ref: ${{ needs.ephemeral-env-label.outputs.sha }}
|
|
persist-credentials: false
|
|
|
|
- name: Setup Docker Environment
|
|
uses: ./.github/actions/setup-docker
|
|
with:
|
|
dockerhub-user: ${{ secrets.DOCKERHUB_USER }}
|
|
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
|
|
build: "true"
|
|
install-docker-compose: "false"
|
|
|
|
- name: Setup supersetbot
|
|
uses: ./.github/actions/setup-supersetbot/
|
|
|
|
- name: Build ephemeral env image
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
supersetbot docker \
|
|
--push \
|
|
--load \
|
|
--preset ci \
|
|
--platform linux/amd64 \
|
|
--context-ref "$RELEASE" \
|
|
--extra-flags "--build-arg INCLUDE_CHROMIUM=false"
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v4
|
|
with:
|
|
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
aws-region: us-west-2
|
|
|
|
- name: Login to Amazon ECR
|
|
id: login-ecr
|
|
uses: aws-actions/amazon-ecr-login@v2
|
|
|
|
- name: Load, tag and push image to ECR
|
|
id: push-image
|
|
env:
|
|
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
|
|
ECR_REPOSITORY: superset-ci
|
|
IMAGE_TAG: apache/superset:${{ needs.ephemeral-env-label.outputs.sha }}-ci
|
|
PR_NUMBER: ${{ github.event.inputs.issue_number || github.event.pull_request.number }}
|
|
run: |
|
|
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-$PR_NUMBER-ci
|
|
docker push -a $ECR_REGISTRY/$ECR_REPOSITORY
|
|
|
|
ephemeral-env-up:
|
|
needs: [ephemeral-env-label, ephemeral-docker-build]
|
|
if: needs.ephemeral-env-label.outputs.slash-command == 'up'
|
|
name: Spin up an ephemeral environment
|
|
runs-on: ubuntu-24.04
|
|
permissions:
|
|
contents: read
|
|
pull-requests: write
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v4
|
|
with:
|
|
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
aws-region: us-west-2
|
|
|
|
- name: Login to Amazon ECR
|
|
id: login-ecr
|
|
uses: aws-actions/amazon-ecr-login@v2
|
|
|
|
- name: Check target image exists in ECR
|
|
id: check-image
|
|
continue-on-error: true
|
|
env:
|
|
PR_NUMBER: ${{ github.event.inputs.issue_number || github.event.pull_request.number }}
|
|
run: |
|
|
aws ecr describe-images \
|
|
--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
|
|
--repository-name superset-ci \
|
|
--image-ids imageTag=pr-$PR_NUMBER-ci
|
|
|
|
- name: Fail on missing container image
|
|
if: steps.check-image.outcome == 'failure'
|
|
uses: actions/github-script@v7
|
|
with:
|
|
github-token: ${{ github.token }}
|
|
script: |
|
|
const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.';
|
|
github.rest.issues.createComment({
|
|
issue_number: ${{ github.event.inputs.issue_number || github.event.pull_request.number }},
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
body: errMsg
|
|
});
|
|
core.setFailed(errMsg);
|
|
|
|
- name: Fill in the new image ID in the Amazon ECS task definition
|
|
id: task-def
|
|
uses: aws-actions/amazon-ecs-render-task-definition@v1
|
|
with:
|
|
task-definition: .github/workflows/ecs-task-definition.json
|
|
container-name: superset-ci
|
|
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.inputs.issue_number || github.event.pull_request.number }}-ci
|
|
|
|
- name: Update env vars in the Amazon ECS task definition
|
|
run: |
|
|
cat <<< "$(jq '.containerDefinitions[0].environment += ${{ needs.ephemeral-env-label.outputs.feature-flags }}' < ${{ steps.task-def.outputs.task-definition }})" > ${{ steps.task-def.outputs.task-definition }}
|
|
|
|
- name: Describe ECS service
|
|
id: describe-services
|
|
run: |
|
|
echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.inputs.issue_number || github.event.pull_request.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
|
|
- name: Create ECS service
|
|
id: create-service
|
|
if: steps.describe-services.outputs.active != 'true'
|
|
env:
|
|
ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
|
|
ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
|
|
PR_NUMBER: ${{ github.event.inputs.issue_number || github.event.pull_request.number }}
|
|
run: |
|
|
aws ecs create-service \
|
|
--cluster superset-ci \
|
|
--service-name pr-$PR_NUMBER-service \
|
|
--task-definition superset-ci \
|
|
--launch-type FARGATE \
|
|
--desired-count 1 \
|
|
--platform-version LATEST \
|
|
--network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \
|
|
--tags key=pr,value=$PR_NUMBER key=github_user,value=${{ github.actor }}
|
|
- name: Deploy Amazon ECS task definition
|
|
id: deploy-task
|
|
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
|
|
with:
|
|
task-definition: ${{ steps.task-def.outputs.task-definition }}
|
|
service: pr-${{ github.event.inputs.issue_number || github.event.pull_request.number }}-service
|
|
cluster: superset-ci
|
|
wait-for-service-stability: true
|
|
wait-for-minutes: 10
|
|
|
|
- name: List tasks
|
|
id: list-tasks
|
|
run: |
|
|
echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.inputs.issue_number || github.event.pull_request.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT
|
|
- name: Get network interface
|
|
id: get-eni
|
|
run: |
|
|
echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks[0].attachments[0].details | map(select(.name==\"networkInterfaceId\"))[0].value')" >> $GITHUB_OUTPUT
|
|
- name: Get public IP
|
|
id: get-ip
|
|
run: |
|
|
echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT
|
|
- name: Comment (success)
|
|
if: ${{ success() }}
|
|
uses: actions/github-script@v7
|
|
with:
|
|
github-token: ${{github.token}}
|
|
script: |
|
|
const issue_number = context.payload.inputs?.issue_number || context.issue.number;
|
|
github.rest.issues.createComment({
|
|
issue_number: issue_number,
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
body: `@${{ github.actor }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are 'admin'/'admin'. Please allow several minutes for bootstrapping and startup.`
|
|
});
|
|
- name: Comment (failure)
|
|
if: ${{ failure() }}
|
|
uses: actions/github-script@v7
|
|
with:
|
|
github-token: ${{github.token}}
|
|
script: |
|
|
const issue_number = context.payload.inputs?.issue_number || context.issue.number;
|
|
github.rest.issues.createComment({
|
|
issue_number: issue_number,
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
|
|
})
|