diff --git a/.github/workflows/bashlib.sh b/.github/workflows/bashlib.sh index c35caa7dc..82840becb 100644 --- a/.github/workflows/bashlib.sh +++ b/.github/workflows/bashlib.sh @@ -129,35 +129,10 @@ cypress-install() { cache-save cypress } -# Run Cypress and upload coverage reports -cypress-run() { +cypress-run-all() { + local USE_DASHBOARD=$1 cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base" - local page=$1 - local group=${2:-Default} - local cypress="./node_modules/.bin/cypress run" - local browser=${CYPRESS_BROWSER:-chrome} - - export TERM="xterm" - export ELECTRON_DISABLE_GPU=true # Attempt to disable GPU for Electron-based Cypress - - say "::group::Run Cypress for [$page]" - if [[ -z $CYPRESS_KEY ]]; then - xvfb-run --auto-servernum --server-args='-screen 0, 1024x768x24' $cypress --spec "cypress/e2e/$page" --browser "$browser" - else - export CYPRESS_RECORD_KEY=$(echo $CYPRESS_KEY | base64 --decode) - # additional flags for Cypress dashboard recording - xvfb-run --auto-servernum --server-args='-screen 0, 1024x768x24' $cypress --spec "cypress/e2e/$page" --browser "$browser" \ - --record --group "$group" --tag "${GITHUB_REPOSITORY},${GITHUB_EVENT_NAME}" \ - --parallel --ci-build-id "${GITHUB_SHA:0:8}-${NONCE}" - - fi - - # don't add quotes to $record because we do want word splitting - say "::endgroup::" -} - -cypress-run-all() { # Start Flask and run it in background # --no-debugger means disable the interactive debugger on the 500 page # so errors can print to stderr. @@ -168,27 +143,17 @@ cypress-run-all() { nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 "$flasklog" 2>&1 None: def main(event_type: str, sha: str, repo: str) -> None: """Main function to check for file changes based on event context.""" print("SHA:", sha) + print("EVENT_TYPE", event_type) if event_type == "pull_request": pr_number = os.getenv("GITHUB_REF", "").split("/")[-2] files = fetch_changed_files_pr(repo, pr_number) @@ -108,13 +109,19 @@ def main(event_type: str, sha: str, repo: str) -> None: files = fetch_changed_files_push(repo, sha) print("Files touched since previous commit:") print_files(files) + + elif event_type == "workflow_dispatch": + print("Workflow dispatched, assuming all changed") + else: raise ValueError("Unsupported event type") changes_detected = {} for group, regex_patterns in PATTERNS.items(): patterns_compiled = [re.compile(p) for p in regex_patterns] - changes_detected[group] = detect_changes(files, patterns_compiled) + changes_detected[group] = event_type == "workflow_dispatch" or detect_changes( + files, patterns_compiled + ) # Output results output_path = os.getenv("GITHUB_OUTPUT") or "/tmp/GITHUB_OUTPUT.txt" diff --git a/scripts/cypress_run.py b/scripts/cypress_run.py new file mode 100644 index 000000000..a04fb861f --- /dev/null +++ b/scripts/cypress_run.py @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import hashlib +import os +import subprocess +from datetime import datetime + +XVFB_PRE_CMD = "xvfb-run --auto-servernum --server-args='-screen 0, 1024x768x24' " +REPO = os.getenv("GITHUB_REPOSITORY") or "apache/superset" +GITHUB_EVENT_NAME = os.getenv("GITHUB_REPOSITORY") or "push" + + +def compute_hash(file_path: str) -> str: + return hashlib.md5(file_path.encode()).hexdigest() + + +def compute_group_index(hash_value: str, num_groups: int) -> int: + return int(hash_value, 16) % num_groups + + +def generate_build_id() -> str: + now = datetime.now() + rounded_minute = now.minute - (now.minute % 20) + rounded_time = now.replace(minute=rounded_minute, second=0, microsecond=0) + return (os.getenv("GITHUB_SHA") or "DUMMY")[:8] + rounded_time.strftime( + "%Y%m%d%H%M" + ) + + +def get_cypress_cmd( + spec_list: list[str], _filter: str, group: str, use_dashboard: bool +) -> str: + cypress_cmd = "./node_modules/.bin/cypress run" + + os.environ["TERM"] = "xterm" + os.environ["ELECTRON_DISABLE_GPU"] = "true" + build_id = generate_build_id() + browser = os.getenv("CYPRESS_BROWSER", "chrome") + + if use_dashboard: + # Run using cypress.io service + cypress_key = os.getenv("CYPRESS_KEY") + command = f"echo {cypress_key} | base64 --decode" + cypress_record_key = ( + subprocess.check_output(command, shell=True).decode("utf-8").strip() + ) + os.environ["CYPRESS_RECORD_KEY"] = cypress_record_key + spec: str = "*/**/*" + cmd = ( + f"{XVFB_PRE_CMD} " + f'{cypress_cmd} --spec "{spec}" --browser {browser} ' + f"--record --group {group} --tag {REPO},{GITHUB_EVENT_NAME} " + f"--parallel --ci-build-id {build_id}" + ) + else: + # Run local, but split the execution + os.environ.pop("CYPRESS_KEY", None) + spec_list_str = ",".join(sorted(spec_list)) + if _filter: + spec_list_str = ",".join(sorted([s for s in spec_list if _filter in s])) + cmd = ( + f"{XVFB_PRE_CMD} " + f"{cypress_cmd} --browser {browser} " + f'--spec "{spec_list_str}" ' + ) + return cmd + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Generate Cypress commands based on test file hash" + ) + parser.add_argument( + "--use-dashboard", + action="store_true", + help="Use Cypress Dashboard for parallelization", + ) + parser.add_argument( + "--parallelism", type=int, default=10, help="Number of parallel groups" + ) + parser.add_argument( + "--parallelism-id", type=int, required=True, help="ID of the parallelism group" + ) + parser.add_argument( + "--filter", type=str, required=False, default=None, help="filter to test" + ) + parser.add_argument("--group", type=str, default="Default", help="Group name") + parser.add_argument( + "--dry-run", + action="store_true", + help="Print the command instead of executing it", + ) + args = parser.parse_args() + + script_dir = os.path.dirname(os.path.abspath(__file__)) + cypress_base_path = "superset-frontend/cypress-base/" + cypress_base_full_path = os.path.join(script_dir, "../", cypress_base_path) + cypress_tests_path = os.path.join(cypress_base_full_path, "cypress/e2e") + + test_files = [] + for root, _, files in os.walk(cypress_tests_path): + for file in files: + if file.endswith("test.ts") or file.endswith("test.js"): + test_files.append( + os.path.join(root, file).replace(cypress_base_full_path, "") + ) + + # Initialize groups + groups: dict[int, list[str]] = {i: [] for i in range(args.parallelism)} + + # Sort test files to ensure deterministic distribution + sorted_test_files = sorted(test_files) + + # Distribute test files in a round-robin manner + for index, test_file in enumerate(sorted_test_files): + group_index = index % args.parallelism + groups[group_index].append(test_file) + + group_id = args.parallelism_id + spec_list = groups[group_id] + cmd = get_cypress_cmd(spec_list, args.filter, args.group, args.use_dashboard) + print(f"RUN: {cmd}") + if not args.dry_run: + subprocess.run(cmd, shell=True, check=True, stdout=None, stderr=None) + + +if __name__ == "__main__": + main() diff --git a/superset/__init__.py b/superset/__init__.py index e68f60ac6..d82ba9600 100644 --- a/superset/__init__.py +++ b/superset/__init__.py @@ -33,10 +33,10 @@ from superset.extensions import ( ) from superset.security import SupersetSecurityManager # noqa: F401 -# All of the fields located here should be considered legacy. The correct way -# to declare "global" dependencies is to define it in extensions.py, -# then initialize it in app.create_app(). These fields will be removed -# in subsequent PRs as things are migrated towards the factory pattern +# All of the fields located here should be considered legacy. The correct way +# to declare "global" dependencies is to define it in extensions.py, +# then initialize it in app.create_app(). These fields will be removed +# in subsequent PRs as things are migrated towards the factory pattern app: Flask = current_app cache = cache_manager.cache conf = LocalProxy(lambda: current_app.config)