iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(release): add github token to changelog script (#12872)

This commit is contained in:
Daniel Vaz Gaspar 2021-02-02 09:24:40 +00:00 committed by GitHub
parent 941d924df3
commit 3ef641dcf1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 47 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
RELEASING/README.md
View File

@ -118,6 +118,8 @@ git push upstream $SUPERSET_GITHUB_BRANCH
Next, update the `CHANGELOG.md` with all the changes that are included in the release.
Make sure the branch has been pushed to `upstream` to ensure the changelog generator
can pick up changes since the previous release.
Change log script requires a github token and will try to use your env var GITHUB_TOKEN.
you can also pass the token using the parameter `--access_token`.
Example:
```bash

77
RELEASING/changelog.py
View File

@ -17,18 +17,22 @@
# pylint: disable=no-value-for-parameter
import csv as lib_csv
import json
import os
import re
import sys
from dataclasses import dataclass
from time import sleep
from typing import Any, Dict, Iterator, List, Optional, Union
from urllib import request
from urllib.error import HTTPError
import click
try:
from github import BadCredentialsException, Github, PullRequest
except ModuleNotFoundError:
print("PyGithub is a required package for this script")
exit(1)
SUPERSET_REPO = "apache/superset"
@dataclass
class GitLog:
@ -60,47 +64,31 @@ class GitChangeLog:
We want to map a git author to a github login, for that we call github's API
"""
def __init__(self, version: str, logs: List[GitLog]) -> None:
def __init__(
self, version: str, logs: List[GitLog], access_token: Optional[str] = None
) -> None:
self._version = version
self._logs = logs
self._github_login_cache: Dict[str, Optional[str]] = {}
self._wait = 10
github_token = access_token or os.environ.get("GITHUB_TOKEN")
self._github = Github(github_token)
self._superset_repo = ""
def _wait_github_rate_limit(self) -> None:
"""
Waits for available rate limit slots on the github API
"""
while True:
rate_limit_payload = self._fetch_github_rate_limit()
if rate_limit_payload["rate"]["remaining"] > 1:
break
print(".", end="", flush=True)
sleep(self._wait)
print()
@staticmethod
def _fetch_github_rate_limit() -> Dict[str, Any]:
"""
Fetches current github rate limit info
"""
with request.urlopen("https://api.github.com/rate_limit") as response:
payload = json.loads(response.read())
return payload
def _fetch_github_pr(self, pr_number: int) -> Dict[str, Any]:
def _fetch_github_pr(self, pr_number: int) -> PullRequest:
"""
Fetches a github PR info
"""
payload = {}
try:
self._wait_github_rate_limit()
with request.urlopen(
"https://api.github.com/repos/apache/superset/pulls/" f"{pr_number}"
) as response:
payload = json.loads(response.read())
except HTTPError as ex:
print(f"{ex}", flush=True)
return payload
github_repo = self._github.get_repo(SUPERSET_REPO)
except BadCredentialsException as ex:
print(
f"Bad credentials to github provided"
f" use access_token parameter or set GITHUB_TOKEN"
)
sys.exit(1)
return github_repo.get_pull(pr_number)
def _get_github_login(self, git_log: GitLog) -> Optional[str]:
"""
@ -113,7 +101,7 @@ class GitChangeLog:
if git_log.pr_number:
pr_info = self._fetch_github_pr(git_log.pr_number)
if pr_info:
github_login = pr_info["user"]["login"]
github_login = pr_info.user.login
else:
github_login = author_name
# set cache
@ -131,7 +119,7 @@ class GitChangeLog:
github_login = log.author
result = result + (
f"- [#{log.pr_number}]"
f"(https://github.com/apache/superset/pull/{log.pr_number}) "
f"(https://github.com/{SUPERSET_REPO}/pull/{log.pr_number}) "
f"{log.message} (@{github_login})\n"
)
print(f"\r {i}/{len(self._logs)}", end="", flush=True)
@ -141,7 +129,7 @@ class GitChangeLog:
for log in self._logs:
yield {
"pr_number": log.pr_number,
"pr_link": f"https://github.com/apache/superset/pull/"
"pr_link": f"https://github.com/{SUPERSET_REPO}/pull/"
f"{log.pr_number}",
"message": log.message,
"time": log.time,
@ -276,13 +264,20 @@ def compare(base_parameters: BaseParameters) -> None:
@click.option(
"--csv", help="The csv filename to export the changelog to",
)
@click.option(
"--access_token",
help="The github access token,"
" if not provided will try to fetch from GITHUB_TOKEN env var",
)
@click.pass_obj
def change_log(base_parameters: BaseParameters, csv: str) -> None:
def change_log(base_parameters: BaseParameters, csv: str, access_token: str) -> None:
""" Outputs a changelog (by PR) """
previous_logs = base_parameters.previous_logs
current_logs = base_parameters.current_logs
previous_diff_logs = previous_logs.diff(current_logs)
logs = GitChangeLog(current_logs.git_ref, previous_diff_logs[::-1])
logs = GitChangeLog(
current_logs.git_ref, previous_diff_logs[::-1], access_token=access_token
)
if csv:
with open(csv, "w") as csv_file:
log_items = list(logs)

6
requirements/base.txt
View File

@ -77,7 +77,7 @@ prison==0.1.3 # via flask-appbuilder
py==1.9.0 # via retry
pyarrow==1.0.1 # via apache-superset
pycparser==2.20 # via cffi
pyjwt==1.7.1 # via flask-appbuilder, flask-jwt-extended
pyjwt==1.7.1 # via apache-superset, flask-appbuilder, flask-jwt-extended
pymeeus==0.3.7 # via convertdate
pyparsing==2.4.7 # via apache-superset, packaging
pyrsistent==0.16.1 # via -r requirements/base.in, jsonschema
@ -92,12 +92,12 @@ redis==3.5.3 # via apache-superset
retry==0.9.2 # via apache-superset
selenium==3.141.0 # via apache-superset
simplejson==3.17.2 # via apache-superset
six==1.15.0 # via bleach, cryptography, flask-jwt-extended, flask-talisman, holidays, isodate, jsonschema, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json
six==1.15.0 # via bleach, cryptography, flask-jwt-extended, flask-talisman, holidays, isodate, jsonschema, packaging, pathlib2, polyline, prison, pyrsistent, python-dateutil, sqlalchemy-utils, wtforms-json
slackclient==2.5.0 # via apache-superset
sqlalchemy-utils==0.36.8 # via apache-superset, flask-appbuilder
sqlalchemy==1.3.20 # via alembic, apache-superset, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
sqlparse==0.3.0 # via apache-superset
typing-extensions==3.7.4.3 # via aiohttp, yarl
typing-extensions==3.7.4.3 # via aiohttp, apache-superset, yarl
urllib3==1.25.11 # via selenium
vine==1.3.0 # via amqp, celery
webencodings==0.5.1 # via bleach

1
requirements/development.in
View File

@ -24,3 +24,4 @@ pyhive[hive]>=0.6.1
psycopg2-binary==2.8.5
tableschema
thrift>=0.11.0,<1.0.0
pygithub>=1.54.1,<2.0.0

7
requirements/development.txt
View File

@ -1,4 +1,4 @@
# SHA1:ae0364cae066a5cb8fb543c4f568bfcdacb6c1b7
# SHA1:b7181d683bed10ffe4892d2f07bc94a503f46b44
#
# This file is autogenerated by pip-compile-multi
# To update, run:
@ -11,6 +11,7 @@ boto3==1.16.10 # via tabulator
botocore==1.19.10 # via boto3, s3transfer
cached-property==1.5.2 # via tableschema
certifi==2020.6.20 # via requests
deprecated==1.2.11 # via pygithub
et-xmlfile==1.0.1 # via openpyxl
flask-cors==3.0.9 # via -r requirements/development.in
future==0.18.2 # via pyhive
@ -24,8 +25,9 @@ openpyxl==3.0.5 # via tabulator
pillow==7.2.0 # via -r requirements/development.in
psycopg2-binary==2.8.5 # via -r requirements/development.in
pydruid==0.6.1 # via -r requirements/development.in
pygithub==1.54.1 # via -r requirements/development.in
pyhive[hive]==0.6.3 # via -r requirements/development.in
requests==2.24.0 # via pydruid, tableschema, tabulator
requests==2.24.0 # via pydruid, pygithub, tableschema, tabulator
rfc3986==1.4.0 # via tableschema
s3transfer==0.3.3 # via boto3
sasl==0.2.1 # via pyhive, thrift-sasl
@ -34,6 +36,7 @@ tabulator==1.52.5 # via tableschema
thrift-sasl==0.4.2 # via pyhive
thrift==0.13.0 # via -r requirements/development.in, pyhive, thrift-sasl
unicodecsv==0.14.1 # via tableschema, tabulator
wrapt==1.12.1 # via deprecated
xlrd==1.2.0 # via tabulator
# The following packages are considered to be unsafe in a requirements file:

1
requirements/testing.txt
View File

@ -40,7 +40,6 @@ traitlets==5.0.5 # via ipython
typed-ast==1.4.1 # via astroid
wcwidth==0.2.5 # via prompt-toolkit
websocket-client==0.57.0 # via docker
wrapt==1.12.1 # via astroid
# The following packages are considered to be unsafe in a requirements file:
# pip