From 0af124eaae945bdc3718b51e59a599d20ab448a4 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Fri, 8 Nov 2024 13:17:55 -0800 Subject: [PATCH] feat: add a script to check environment software versions (#29609) --- Dockerfile | 4 +- docker-compose.yml | 18 +++ pyproject.toml | 1 + requirements/development.txt | 14 ++- scripts/check-env.py | 222 +++++++++++++++++++++++++++++++++++ 5 files changed, 257 insertions(+), 2 deletions(-) create mode 100755 scripts/check-env.py diff --git a/Dockerfile b/Dockerfile index 229d21747..3b9273cb7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -85,10 +85,11 @@ RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \ RUN rm /app/superset/translations/*/LC_MESSAGES/*.po RUN rm /app/superset/translations/messages.pot +FROM python:${PY_VER} AS python-base ###################################################################### # Final lean image... ###################################################################### -FROM python:${PY_VER} AS lean +FROM python-base AS lean # Include translations in the final build. The default supports en only to # reduce complexity and weight for those only using en @@ -120,6 +121,7 @@ COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./ # setup.py uses the version information in package.json COPY --chown=superset:superset superset-frontend/package.json superset-frontend/ COPY --chown=superset:superset requirements/base.txt requirements/ +COPY --chown=superset:superset scripts/check-env.py scripts/ RUN --mount=type=cache,target=/root/.cache/pip \ apt-get update -qq && apt-get install -yqq --no-install-recommends \ build-essential \ diff --git a/docker-compose.yml b/docker-compose.yml index 605be1333..e8dc2efa8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,7 @@ x-superset-user: &superset-user root x-superset-depends-on: &superset-depends-on - db - redis + - superset-checks x-superset-volumes: &superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container - ./docker:/app/docker @@ -130,6 +131,23 @@ services: - REDIS_PORT=6379 - REDIS_SSL=false + superset-checks: + build: + context: . + target: python-base + cache_from: + - apache/superset-cache:3.10-slim-bookworm + container_name: superset_checks + command: ["/app/scripts/check-env.py"] + env_file: + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false + user: *superset-user + healthcheck: + disable: true + superset-init: build: <<: *common-build diff --git a/pyproject.toml b/pyproject.toml index fd603af6a..551e12bc1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -188,6 +188,7 @@ development = [ "pip-compile-multi", "pre-commit", "progress>=1.5,<2", + "psutil", "pyfakefs", "pyinstrument>=4.0.2,<5", "pylint", diff --git a/requirements/development.txt b/requirements/development.txt index b2ed1b8b3..b8c93cca5 100644 --- a/requirements/development.txt +++ b/requirements/development.txt @@ -174,6 +174,8 @@ protobuf==4.23.0 # googleapis-common-protos # grpcio-status # proto-plus +psutil==6.0.0 + # via apache-superset psycopg2-binary==2.9.6 # via apache-superset pure-sasl==0.6.2 @@ -186,7 +188,7 @@ pyee==11.0.1 # via playwright pyfakefs==5.3.5 # via apache-superset -pyhive[hive_pure_sasl]==0.7.0 +pyhive[presto]==0.7.0 # via apache-superset pyinstrument==4.4.0 # via apache-superset @@ -233,6 +235,16 @@ thrift==0.16.0 # thrift-sasl thrift-sasl==0.4.3 # via apache-superset +tomli==2.0.1 + # via + # build + # coverage + # pip-tools + # pylint + # pyproject-api + # pyproject-hooks + # pytest + # tox tomlkit==0.12.5 # via pylint toposort==1.10 diff --git a/scripts/check-env.py b/scripts/check-env.py new file mode 100755 index 000000000..647aa1142 --- /dev/null +++ b/scripts/check-env.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import platform +import subprocess +import sys +from typing import Callable, Optional, Set, Tuple + +import click +import psutil +from packaging.version import InvalidVersion, Version + + +class Requirement: + def __init__( + self, + name: str, + ideal_range: Tuple[Version, Version], + supported_range: Tuple[Version, Version], + req_type: str, + command: str, + version_post_process: Optional[Callable[[str], str]] = None, + ): + self.name = name + self.ideal_range = ideal_range + self.supported_range = supported_range + self.req_type = req_type + self.command = command + self.version_post_process = version_post_process + self.version = self.get_version() + self.status = self.check_version() + + def get_version(self) -> Optional[str]: + try: + version = subprocess.check_output(self.command, shell=True).decode().strip() + if self.version_post_process: + version = self.version_post_process(version) + return version.split()[-1] + except subprocess.CalledProcessError: + return None + + def check_version(self) -> str: + if self.version is None: + return "❌ Not Installed" + + try: + version_number = Version(self.version) + except InvalidVersion: + return "❌ Invalid Version Format" + + ideal_min, ideal_max = self.ideal_range + supported_min, supported_max = self.supported_range + + if ideal_min <= version_number <= ideal_max: + return "✅ Ideal" + elif supported_min <= version_number: + return "🟡 Supported" + else: + return "❌ Unsupported" + + def format_result(self) -> str: + ideal_range_str = f"{self.ideal_range[0]} - {self.ideal_range[1]}" + supported_range_str = f"{self.supported_range[0]} - {self.supported_range[1]}" + return f"{self.status.split()[0]} {self.name:<25} {self.version or 'N/A':<25} {ideal_range_str:<25} {supported_range_str:<25}" + + +def check_memory(min_gb: int) -> str: + total_memory = psutil.virtual_memory().total / (1024**3) + if total_memory >= min_gb: + return f"✅ Memory: {total_memory:.2f} GB" + else: + return f"❌ Memory: {total_memory:.2f} GB (Minimum required: {min_gb} GB)" + + +def get_cpu_info() -> str: + cpu_count = psutil.cpu_count(logical=True) + cpu_freq = psutil.cpu_freq() + cpu_info = ( + f"{cpu_count} cores at {cpu_freq.current:.2f} MHz" + if cpu_freq + else f"{cpu_count} cores" + ) + return f"CPU: {cpu_info}" + + +def get_docker_platform() -> str: + try: + output = ( + subprocess.check_output( + "docker info --format '{{.OperatingSystem}}'", shell=True + ) + .decode() + .strip() + ) + if "Docker Desktop" in output: + return f"Docker Platform: {output} ({platform.system()})" + return f"Docker Platform: {output}" + except subprocess.CalledProcessError: + return "Docker Platform: ❌ Not Detected" + + +@click.command( + help=""" +This script checks the local environment for various software versions and other requirements, providing feedback on whether they are ideal, supported, or unsupported. +""" +) +@click.option( + "--docker", is_flag=True, help="Check Docker and Docker Compose requirements" +) +@click.option( + "--frontend", + is_flag=True, + help="Check frontend requirements (npm, Node.js, memory)", +) +@click.option("--backend", is_flag=True, help="Check backend requirements (Python)") +def main(docker: bool, frontend: bool, backend: bool) -> None: + requirements = [ + Requirement( + "python", + (Version("3.10.0"), Version("3.10.999")), + (Version("3.9.0"), Version("3.11.999")), + "backend", + "python --version", + ), + Requirement( + "npm", + (Version("10.0.0"), Version("999.999.999")), + (Version("10.0.0"), Version("999.999.999")), + "frontend", + "npm -v", + ), + Requirement( + "node", + (Version("20.0.0"), Version("20.999.999")), + (Version("20.0.0"), Version("20.999.999")), + "frontend", + "node -v", + ), + Requirement( + "docker", + (Version("20.10.0"), Version("999.999.999")), + (Version("19.0.0"), Version("999.999.999")), + "docker", + "docker --version", + lambda v: v.split(",")[0], + ), + Requirement( + "docker-compose", + (Version("2.28.0"), Version("999.999.999")), + (Version("1.29.0"), Version("999.999.999")), + "docker", + "docker-compose --version", + ), + Requirement( + "git", + (Version("2.30.0"), Version("999.999.999")), + (Version("2.20.0"), Version("999.999.999")), + "backend", + "git --version", + ), + ] + + print("==================") + print("System Information") + print("==================") + print(f"OS: {platform.system()} {platform.release()}") + print(get_cpu_info()) + print(get_docker_platform()) + print("\n") + + check_req_types: Set[str] = set() + if docker: + check_req_types.add("docker") + if frontend: + check_req_types.add("frontend") + if backend: + check_req_types.add("backend") + if not check_req_types: + check_req_types.update(["docker", "frontend", "backend"]) + + headers = ["Status", "Software", "Version Found", "Ideal Range", "Supported Range"] + row_format = "{:<2} {:<25} {:<25} {:<25} {:<25}" + + print("=" * 100) + print(row_format.format(*headers)) + print("=" * 100) + + all_ok = True + for requirement in requirements: + if requirement.req_type in check_req_types: + result = requirement.format_result() + if "❌" in requirement.status: + all_ok = False + print(result) + + if "frontend" in check_req_types: + memory_check = check_memory(12) + if "❌" in memory_check: + all_ok = False + print(memory_check) + + if not all_ok: + sys.exit(1) + + +if __name__ == "__main__": + main()