diff --git a/.github/scripts/freertos_zipper.py b/.github/scripts/packager.py similarity index 86% rename from .github/scripts/freertos_zipper.py rename to .github/scripts/packager.py index a46271ecf3..3d9379189e 100755 --- a/.github/scripts/freertos_zipper.py +++ b/.github/scripts/packager.py @@ -100,17 +100,42 @@ def unzip_baseline_zip(path_inzip, path_outdir): return os.path.join(path_outdir, str(os.path.basename(path_inzip)).replace('.zip', '')) -def download_git_tree(git_link, root_dir, dir_name, ref='master', commit_id='HEAD'): +def download_git_tree(git_link, root_dir, dir_name, ref='master', commit_id='HEAD', recurse=False): ''' Download HEAD from Git Master. Place into working files dir ''' - rc = subprocess.run(['git', '-C', root_dir, 'clone', '-b', ref, git_link, dir_name]).returncode - rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'checkout', '-f', commit_id]).returncode - rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'clean', '-fd']).returncode - rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'submodule', 'update', '--init', '--recursive']).returncode + args = ['git', '-C', root_dir, 'clone', '-b', ref, git_link, dir_name] + rc = subprocess.run(args).returncode + rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'checkout', '-f', commit_id]).returncode + rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'clean', '-fd']).returncode + if recurse: + rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'submodule', 'update', '--init', '--recursive']).returncode return os.path.join(root_dir, dir_name) if rc == 0 else None + +def commit_git_tree_changes(repo_dir, commit_message=''): + rc = subprocess.run(['git', '-C', repo_dir, 'add', '-u']).returncode + rc += subprocess.run(['git', '-C', repo_dir, 'commit', '-m', commit_message]).returncode + + return rc + +def push_git_tree_changes(repo_dir, tag=None, force_tag=False): + rc = subprocess.run(['git', '-C', repo_dir, 'push']).returncode + + if tag != None: + force_tag_arg = '-f' if force_tag else '' + rc += subprocess.run(['git', '-C', repo_dir, 'tag', force_tag_arg, tag]).returncode + rc += subprocess.run(['git', '-C', repo_dir, 'push', force_tag_arg, '--tags']).returncode + + return rc + +def update_submodule_pointer(repo_dir, rel_submodule_path, new_submodule_ref): + rc = subprocess.run(['git', '-C', repo_dir, 'submodule', 'update', '--init']).returncode + rc += subprocess.run(['git', '-C', os.path.join(repo_dir, rel_submodule_path), 'fetch']).returncode + rc += subprocess.run(['git', '-C', os.path.join(repo_dir, rel_submodule_path), 'checkout', new_submodule_ref]).returncode + rc += subprocess.run(['git', '-C', repo_dir, 'add', rel_submodule_path]).returncode + def setup_intermediate_files(scratch_dir, intree_dir, outtree_dir): cleanup_intermediate_files(scratch_dir) os.mkdir(scratch_dir) @@ -157,7 +182,7 @@ def zip_result_tree(path_tree, path_outzip): Zip file tree rooted at 'path_root', using same compression as 7z at max compression, to zip at 'path_outzip' ''' - subprocess.run(['7z', 'a', '-tzip', '-mx=9', path_outzip, os.path.join('.', path_tree)]) + subprocess.run(['7z', 'a', '-tzip', '-mx=9', path_outzip, os.path.join('.', path_tree, '*')]) def show_package_diagnostics(path_newzip, path_basezip): ''' @@ -171,13 +196,13 @@ def show_package_diagnostics(path_newzip, path_basezip): path_basezip, '+' if size_diff_KB >= 0 else '', size_diff_KB)) -def create_package(path_outtree, package_name, exclude_files=[]): +def create_package(path_ziproot, path_outtree, package_name, exclude_files=[]): print("Packaging '%s'..." % package_name) pruned_files = prune_result_tree(path_outtree, exclude_files) print('Files removed:\n %s' % '\n '.join(pruned_files)) path_outzip = '%s.zip' % package_name - zip_result_tree(path_outtree, path_outzip) + zip_result_tree(path_ziproot, path_outzip) print('Done.') return path_outzip diff --git a/.github/scripts/release.py b/.github/scripts/release.py new file mode 100755 index 0000000000..0706590ee4 --- /dev/null +++ b/.github/scripts/release.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +import os, shutil +from argparse import ArgumentParser + +import re +import datetime +from github import Github +from github.GithubException import * +from github.InputGitAuthor import InputGitAuthor + +from versioning import update_version_number_in_freertos_component +from versioning import update_freertos_version_macros + +from packager import download_git_tree +from packager import update_submodule_pointer +from packager import commit_git_tree_changes +from packager import push_git_tree_changes +from packager import create_package +from packager import RELATIVE_FILE_EXCLUDES as FREERTOS_RELATIVE_FILE_EXCLUDES + +# PyGithub Git - https://github.com/PyGithub/PyGithub +# PyGithub Docs - https://pygithub.readthedocs.io/en/latest/github_objects +# REST API used by PyGithub - https://developer.github.com/v3/ + +''' +FUTURE ENHANCEMENTS + - Add mechanism that restores state of all affected to repos to BEFORE this script was run + - Input sanitizing + - Include regex patterns that MUST be honored for version strings, etc. + - Create a companion dependencies file that can be piped to pip3 + - Ease of HTTPS authentication + - This should make it very easy to port to Github action. Currently, Github action mostly operates with + via https endpoints, rather than SSH + - Break down some functions and any repeated work --> more granular (reasonably), less duplicated code + - Unit tests + - Theres already an option and some desired tests laid out via comments. See bottom + - All of the scratch-work directories/files should be placed under a single directory the name of which makes obvious + that it's a scratch-work dir (Ex. tmp-*, scratch-*, etc.) + - Intermediate checks + - +''' + +def info(msg, indent_level=0): + print('%s[INFO]: %s' % (' ' * indent_level, str(msg))) + +def warning(msg, indent_level=0): + print('%s[WARNING]: %s' % (' ' * indent_level, str(msg))) + +def error(msg, indent_level=0): + print('%s[ERROR]: %s' % (' ' * indent_level, str(msg))) + +def debug(msg, indent_level=0): + print('%s[DEBUG]: %s' % (' ' * indent_level, str(msg))) + +class BaseRelease: + def __init__(self, mGit, version, commit, git_ssh=False, git_org='FreeRTOS'): + self.version = version + self.tag_msg = 'Autocreated by FreeRTOS Git Tools.' + self.commit = commit + self.git_ssh = git_ssh + self.git_org = git_org + self.commit_msg_prefix = '[AUTO][RELEASE]: ' + + self.mGit = mGit # Save a handle to the authed git session + + def updateFileHeaderVersions(self): + ''' + Updates for all FreeRTOS/FreeRTOS files, not including submodules, to have their file header + versions updated to match this release version. It creates the release tag and stores these updates there, + at a detached commit (akin to a branch). + ''' + assert False, 'Implement me' + + def CheckRelease(self): + ''' + Sanity checks for the release. Verify version number format. Check zip size. + Ensure version numbers were updated, etc. + ''' + assert False, 'Add release check' + + def hasTag(self, tag): + remote_tags = self.repo.get_tags() + for t in remote_tags: + if t.name == tag: + return True + + return False + + def getRemoteEndpoint(self, repo_name): + if self.git_ssh: + return 'git@github.com:%s.git' % repo_name + else: + return 'https://github.com/%s.git' % repo_name + + def printReleases(self): + releases = self.repo.get_releases() + for r in releases: + print(r) + +class KernelRelease(BaseRelease): + def __init__(self, mGit, version, commit, git_ssh=False, git_org='FreeRTOS'): + super().__init__(mGit, version, commit, git_ssh=git_ssh, git_org=git_org) + + self.repo_name = '%s/FreeRTOS-Kernel' % self.git_org + self.repo = mGit.get_repo(self.repo_name) + self.tag = 'V%s' % version + + def updateFileHeaderVersions(self, old_version_prefix): + ''' + Adds changes for two commits + 1.) Updates to file headers + 2.) Update to task.h macros + Then tags commit #2 with the new tag version. Notes this will overwrite a tag it already exists + Finally pushes all these changes + ''' + remote_name = self.getRemoteEndpoint(self.repo_name) + rel_repo_path = 'tmp-versioning-freertos-kernel' + + # Clean up any old work from previous runs + if os.path.exists(rel_repo_path): + shutil.rmtree(rel_repo_path) + + # Download master:HEAD. Update its file header versions and kernel macros + repo_path = download_git_tree(remote_name, '.', rel_repo_path, 'master', 'HEAD') + assert repo_path != None, 'Failed to download git tree' + + update_version_number_in_freertos_component(repo_path, '.', old_version_prefix, 'FreeRTOS Kernel V%s' % self.version) + commit_git_tree_changes(rel_repo_path, commit_message=self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version) + + (major, minor, build) = self.version.split('.') + update_freertos_version_macros(os.path.join(repo_path, 'include', 'task.h'), major, minor, build) + commit_git_tree_changes(rel_repo_path, commit_message=self.commit_msg_prefix + 'Bump task.h version macros to "%s"' % self.version) + + # Commit the versioning, tag it, and upload all to remote + rc = push_git_tree_changes(repo_path, tag=self.tag, force_tag=True) + assert rc == 0, 'Failed to upload git tree changes' + + +class FreertosRelease(BaseRelease): + def __init__(self, mGit, version, commit, git_ssh=False, git_org='FreeRTOS'): + super().__init__(mGit, version, commit, git_ssh=git_ssh, git_org=git_org) + + self.repo_name = '%s/FreeRTOS' % self.git_org + self.repo = mGit.get_repo(self.repo_name) + self.tag = self.version + self.description = 'Contains source code and example projects for the FreeRTOS Kernel and FreeRTOS+ libraries.' + self.zip = None + + def updateFileHeaderVersions(self, old_version_prefix, new_kernel_ref): + remote_name = self.getRemoteEndpoint(self.repo_name) + rel_repo_path = 'tmp-versioning-freertos' + + # Clean up any old work from previous runs + if os.path.exists(rel_repo_path): + shutil.rmtree(rel_repo_path) + + # Download master:HEAD. Update its file header versions and kernel submodule pointer + repo_path = download_git_tree(remote_name, '.', rel_repo_path, 'master', 'HEAD') + assert repo_path != None, 'Failed to download git tree' + + update_version_number_in_freertos_component(repo_path, '.', old_version_prefix, 'FreeRTOS V%s' % self.version) + commit_git_tree_changes(repo_path, commit_message=self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version) + + update_submodule_pointer(repo_path, os.path.join('FreeRTOS', 'Source'), new_kernel_ref) + commit_git_tree_changes(repo_path, commit_message=self.commit_msg_prefix + 'Bump kernel pointer "%s"' % new_kernel_ref) + + # Commit the versioning, tag it, and upload all to remote + rc = push_git_tree_changes(repo_path, tag=self.tag, force_tag=True) + assert rc == 0, 'Failed to upload git tree changes' + + def CreateReleaseZip(self): + ''' + At the moment, the only asset we upload is the + ''' + remote_name = self.getRemoteEndpoint(self.repo_name) + + # This path name is retained in zip, so we don't name it 'tmp-*' but rather keep it consistent with previous + # packaging + repo_name = 'FreeRTOSv%s' % self.version + zip_root_path = repo_name + rel_repo_path = os.path.join(zip_root_path, repo_name) + + # Clean up any old work from previous runs + if os.path.exists(zip_root_path): + shutil.rmtree(zip_root_path) + + # To keep consistent with previous packages + os.mkdir(zip_root_path) + + # Download master:HEAD. Update its file header versions and kernel submodule pointer + repo_path = download_git_tree(remote_name, '.', rel_repo_path, 'master', self.tag, recurse=True) + assert repo_path != None, 'Failed to download git tree' + + self.zip = create_package(zip_root_path, + rel_repo_path, + 'FreeRTOSv%s' % self.version, + exclude_files=FREERTOS_RELATIVE_FILE_EXCLUDES) + + def Upload(self): + ''' + Creates/Overwrites release identified by target tag + ''' + + # If this release already exists, delete it + try: + release_queried = self.repo.get_release(self.tag) + + info('Deleting existing release "%s"...' % self.tag) + release_queried.delete_release() + except UnknownObjectException: + info('Creating release/tag "%s"...' % self.tag) + + # Create the new release endpoind at upload assets + release = self.repo.create_git_release(tag = self.tag, + name = 'FreeRTOSv%s' % (self.version), + message = self.description, + draft = False, + prerelease = False) + + release.upload_asset(self.zip, name='FreeRTOSv%s.zip' % self.version, content_type='application/zip') + + +def configure_argparser(): + parser = ArgumentParser(description='FreeRTOS Release tool') + + parser.add_argument('--old-core-version', + default=None, + required=True, + help='FreeRTOS Version to match and replace. (Ex. FreeRTOS V202011.00)') + + parser.add_argument('--new-core-version', + default=None, + required=True, + help='FreeRTOS Version to replace old version. (Ex. FreeRTOS V202011.00)') + + parser.add_argument('--old-kernel-version', + default=None, + required=True, + help='FreeRTOS-Kernel Version to match and replace. (Ex. "FreeRTOS Kernel V10.4.1")') + + parser.add_argument('--new-kernel-version', + default=None, + required=True, + help='FreeRTOS-Kernel Version to replace old version. (Ex. "FreeRTOS Kernel V10.4.1")') + + parser.add_argument('--git-org', + default='FreeRTOS', + required=False, + help='Git organization owner for FreeRTOS and FreeRTOS-Kernel. (i.e. "/FreeRTOS.git")') + + parser.add_argument('--use-git-ssh', + default=False, + action='store_true', + help='Use SSH endpoints to interface git remotes, instead of HTTPS') + + + parser.add_argument('--unit-test', + action='store_true', + default=False, + help='Run unit tests.') + + return parser + +def sanitize_cmd_args(args): + info('TODO: Add cmdline input sanitizing') + +def main(): + # CLI + cmd = configure_argparser() + + # Setup + args = cmd.parse_args() + sanitize_cmd_args(args) + + # Auth + assert 'GITHUB_TOKEN' in os.environ, 'You must set env variable GITHUB_TOKEN to an authorized git PAT' + mGit = Github(os.environ.get('GITHUB_TOKEN')) + + if args.unit_test: + pass + else: + # Update versions + rel_kernel = KernelRelease(mGit, args.new_kernel_version, None, git_ssh=args.use_git_ssh, git_org=args.git_org) + rel_kernel.updateFileHeaderVersions(args.old_kernel_version) + + rel_freertos = FreertosRelease(mGit, args.new_core_version, None, git_ssh=args.use_git_ssh, git_org=args.git_org) + rel_freertos.updateFileHeaderVersions(args.old_core_version, 'V%s' % args.new_kernel_version) + + # Package contents of FreeRTOS/FreeRTOS and upload release assets to Git + rel_freertos.CreateReleaseZip() + rel_freertos.Upload() + + info('Review script output for any unexpected behaviour.') + info('Release done.') + +if __name__ == '__main__': + main() + +#-------------------------------------------------------------------- +# TESTING +#-------------------------------------------------------------------- +# Create new tag, verify creation + +# Create release endpoint, delete it, verify deletion + +# Overwrite an existing tag + +# Perform full operation, restore to state before operation, verify restored state + +# Run zipping operation, check versions, pathing, etc diff --git a/.github/scripts/frog_version_number_update.py b/.github/scripts/versioning.py similarity index 99% rename from .github/scripts/frog_version_number_update.py rename to .github/scripts/versioning.py index e06c4a31aa..7beec75d7f 100755 --- a/.github/scripts/frog_version_number_update.py +++ b/.github/scripts/versioning.py @@ -224,7 +224,7 @@ def update_freertos_version_macros(path_macrofile, major, minor, build): print('Done. Replaced "%s" --> "V%s.%s.%s".' % (old_version_number, major, minor, build)) -def update_version_number_in_freertos_component(component, root_dir, old_version, new_version, verbose=False): +def update_version_number_in_freertos_component(component, root_dir, old_version_prefix, new_version, verbose=False): print('Updating "%s"...' % component) component_files = list_files_in_a_component(component, root_dir, ext_filter=None) version_numbers = defaultdict(list) @@ -242,7 +242,7 @@ def update_version_number_in_freertos_component(component, root_dir, old_version old_version_string = vkey[0] new_version_string = new_version - if old_version in old_version_string: + if old_version_prefix in old_version_string: if old_version_string != new_version_string: files_using_old_version = version_numbers[vkey]