#!/usr/bin/env python3 import os, shutil from yaml import load, dump try: from yaml import CLoader as Loader, CDumper as Dumper except ImportError: from yaml import Loader, Dumper from argparse import ArgumentParser # For interfacing Git REST API import re import datetime from github import Github from github.GithubException import * from github.InputGitAuthor import InputGitAuthor # Local interfacing of repo from git import Repo from git import PushInfo import zipfile from versioning import update_version_number_in_freertos_component from versioning import update_freertos_version_macros from packager import prune_result_tree, prune_result_tree_v2 from packager import RELATIVE_FILE_EXCLUDES as FREERTOS_RELATIVE_FILE_EXCLUDES # PyGithub Git - https://github.com/PyGithub/PyGithub # PyGithub Docs - https://pygithub.readthedocs.io/en/latest/github_objects # REST API used by PyGithub - https://developer.github.com/v3/ indent_level = 0 # Files/Directories not to be included in the FreeRTOS-Kernel release. KERNEL_RELEASE_EXCLUDE_FILES = [ '.git', '.github', '.gitignore', '.gitmodules' ] def logIndentPush(): global indent_level indent_level += 4 def logIndentPop(): global indent_level indent_level -= 4 if indent_level < 0: indent_level = 0 def info(msg, end='\n'): print('[INFO]: %s%s' % (' ' * indent_level, str(msg)), end=end, flush=True) def warning(msg): print('[WARNING]: %s%s' % (' ' * indent_level, str(msg)), flush=True) def error(msg): print('[ERROR]: %s%s' % (' ' * indent_level, str(msg)), flush=True) def debug(msg): print('[DEBUG]: %s%s' % (' ' * indent_level, str(msg)), flush=True) # Callback for progress updates. For long spanning gitpython commands def printDot(op_code, cur_count, max_count=None, message=''): if max_count == None or cur_count == max_count: print('.', end='') class BaseRelease: def __init__(self, mGit, version, commit='HEAD', git_ssh=False, git_org='FreeRTOS', repo_path=None, branch='main', do_not_push=False): self.version = version self.tag_msg = 'Autocreated by FreeRTOS Git Tools.' self.commit = commit self.git_ssh = git_ssh self.git_org = git_org self.repo_path = repo_path self.local_repo = None self.branch = branch self.commit_msg_prefix = '[AUTO][RELEASE]: ' self.description = '' self.mGit = mGit # Save a handle to the authed git session self.do_not_push = do_not_push if self.repo_path: info('Sourcing "%s" to make local commits' % self.repo_path) self.local_repo = Repo(self.repo_path) def CheckRelease(self): ''' Sanity checks for the release. Verify version number format. Check zip size. Ensure version numbers were updated, etc. ''' assert False, 'Add release check' def hasTag(self, tag): remote_tags = self.repo.get_tags() for t in remote_tags: if t.name == tag: return True return False def commitChanges(self, msg): assert self.local_repo != None, 'Failed to commit. Git repo uninitialized.' info('Committing: "%s"' % msg) self.local_repo.git.add(update=True) commit = self.local_repo.index.commit(msg) def getRemoteEndpoint(self, repo_name): if self.git_ssh: return 'git@github.com:%s.git' % repo_name else: return 'https://github.com/%s.git' % repo_name def printReleases(self): releases = self.repo.get_releases() for r in releases: print(r) def pushLocalCommits(self, force=False): if self.do_not_push: info('Skipping to push local commits...') else: info('Pushing local commits...') push_infos = self.local_repo.remote('origin').push(force=force) # Check for any errors for push_info in push_infos: assert 0 == push_info.flags & PushInfo.ERROR, 'Failed to push changes to ' + str(push_info) def pushTag(self): if self.do_not_push: info('Skipping to push tag "%s"' % self.tag) else: # Overwrite existing tags info('Pushing tag "%s"' % self.tag) tag_info = self.local_repo.create_tag(self.tag, message=self.tag_msg, force=True) self.local_repo.git.push(tags=True, force=True) def deleteTag(self): # Remove from remote if self.tag in self.local_repo.tags: info('Deleting tag "%s"' % self.tag) self.local_repo.remote('origin').push(':%s' % self.tag) else: info('A tag does not exists for "%s". No need to delete.' % self.tag) def updateSubmodulePointer(self, rel_path, ref): submodule = Repo(rel_path) submodule.remote('origin').fetch() submodule.git.checkout(ref) def updateFileHeaderVersions(self, old_version_substrings, new_version_string): info('Updating file header versions for "%s"...' % self.version, end='') n_updated = 0 n_updated += update_version_number_in_freertos_component(self.repo_path, '.', old_version_substrings, new_version_string, exclude_hidden=True) n_updated += update_version_number_in_freertos_component(os.path.join('.github', 'scripts'), self.repo_path, old_version_substrings, new_version_string, exclude_hidden=False) print('...%d Files updated.' % n_updated) self.commitChanges(self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version) def deleteGitRelease(self): info('Deleting git release endpoint for "%s"' % self.tag) try: self.repo.get_release(self.tag).delete_release() except UnknownObjectException: info('A release endpoint does not exist for "%s". No need to erase.' % self.tag) except: assert False, 'Encountered error while trying to delete git release endpoint' def rollbackAutoCommits(self, n_autocommits=2, n_search=25): info('Rolling back "%s" autocommits' % self.tag) if self.tag not in self.local_repo.tags: error('Could not find a SHA to rollback to for tag "%s"' % self.tag) return False # Search for auto release SHAs that match the release tag SHA tag_commit = self.local_repo.tag('refs/tags/%s' % self.tag).commit prior_commit = self.local_repo.commit(tag_commit.hexsha + '~%d' % n_autocommits) n_commits_searched = 0 for commit in self.local_repo.iter_commits(): if n_commits_searched > n_search: error('Exhaustively searched but could not find tag commit to rollback') return False if (self.commit_msg_prefix in commit.message and commit.hexsha == tag_commit.hexsha and self.version in commit.message): info('Found matching tag commit %s. Reverting to prior commit %s' % (tag_commit.hexsha, prior_commit.hexsha)) # Found the commit prior to this autorelease. Revert back to it then push self.local_repo.git.reset(prior_commit.hexsha, hard=True) self.pushLocalCommits(force=True) return True n_commits_searched += 1 return False def restorePriorToRelease(self): info('Restoring "main" to just before autorelease:%s' % self.version) self.deleteGitRelease() self.rollbackAutoCommits() self.deleteTag() self.pushLocalCommits(force=True) class KernelRelease(BaseRelease): def __init__(self, mGit, version, commit='HEAD', git_ssh=False, git_org='FreeRTOS', repo_path=None, branch='main', main_br_version='', do_not_push=False): super().__init__(mGit, version, commit=commit, git_ssh=git_ssh, git_org=git_org, repo_path=repo_path, branch=branch, do_not_push=do_not_push) self.repo_name = '%s/FreeRTOS-Kernel' % self.git_org self.repo = mGit.get_repo(self.repo_name) self.tag = 'V%s' % version self.description = 'Contains source code for the FreeRTOS Kernel.' self.zip_path = 'FreeRTOS-KernelV%s.zip' % self.version self.main_br_version = main_br_version # Parent ctor configures local_repo if caller chooses to source local repo from repo_path. if self.repo_path is None: self.repo_path = 'tmp-release-freertos-kernel' if os.path.exists(self.repo_path): shutil.rmtree(self.repo_path) # Clone the target repo for creating the release autocommits remote_name = self.getRemoteEndpoint(self.repo_name) info('Downloading %s@%s to baseline auto-commits...' % (remote_name, commit), end='') self.local_repo = Repo.clone_from(remote_name, self.repo_path, progress=printDot, branch=self.branch) # In case user gave non-HEAD commit to baseline self.local_repo.git.checkout(commit) print() def createReleaseZip(self): ''' At the moment, the only asset we upload is the source code. ''' zip_name = 'FreeRTOS-KernelV%s' % self.version info('Packaging "%s"' % zip_name) logIndentPush() # This path name is retained in zip, so we don't name it 'tmp-*' but # rather keep it consistent. rel_repo_path = zip_name # Clean up any old work from previous runs. if os.path.exists(rel_repo_path): shutil.rmtree(rel_repo_path) # Download a fresh copy for packaging. info('Downloading fresh copy of %s for packing...' % zip_name, end='') packaged_repo = Repo.clone_from(self.getRemoteEndpoint(self.repo_name), rel_repo_path, multi_options=['--depth=1', '-b%s' % self.tag, '--recurse-submodules'], progress=printDot, branch=self.branch) print() # Prune then zip package. info('Pruning from release zip...', end='') files_pruned = prune_result_tree_v2(rel_repo_path, KERNEL_RELEASE_EXCLUDE_FILES) print('...%d Files Removed.' % len(files_pruned)) info('Compressing "%s"...' % self.zip_path) with zipfile.ZipFile(self.zip_path, 'w', zipfile.ZIP_DEFLATED, compresslevel=9) as zip: for root, dirs, files in os.walk(rel_repo_path): for file in files: # For some strange reason, we have broken symlinks...avoid these. file_path = os.path.join(root, file) if os.path.islink(file_path) and not os.path.exists(file_path): warning('Skipping over broken symlink "%s"' % file_path) else: zip.write(file_path) logIndentPop() def createGitRelease(self): ''' Creates/Overwrites release identified by target tag ''' if self.do_not_push: info('Skipping creating git release endpoint for "%s"...' % self.tag) else: # If this release already exists, delete it try: release_queried = self.repo.get_release(self.tag) info('Overwriting existing git release endpoint for "%s"...' % self.tag) release_queried.delete_release() except UnknownObjectException: info('Creating git release endpoint for "%s"...' % self.tag) # Create the release asset to upload. self.createReleaseZip() # Create the new release endpoint at upload assets release = self.repo.create_git_release(tag = self.tag, name = 'V%s' % (self.version), message = self.description, draft = False, prerelease = False) info('Uploading release asssets...') release.upload_asset(self.zip_path, name='FreeRTOS-KernelV%s.zip' % self.version, content_type='application/zip') def autoRelease(self): info('Auto-releasing FreeRTOS Kernel V%s' % self.version) self.pushTag() self.createGitRelease() info('Kernel release done.') class FreertosRelease(BaseRelease): def __init__(self, mGit, version, commit, git_ssh=False, git_org='FreeRTOS', repo_path=None, branch='main', do_not_push=False): super().__init__(mGit, version, commit, git_ssh=git_ssh, git_org=git_org, repo_path=repo_path, branch=branch, do_not_push=do_not_push) self.repo_name = '%s/FreeRTOS' % self.git_org self.repo = mGit.get_repo(self.repo_name) self.tag = self.version self.description = 'Contains source code and example projects for the FreeRTOS Kernel and FreeRTOS+ libraries.' self.zip_path = 'FreeRTOSv%s.zip' % self.version # Download a fresh copy of local repo for making autocommits, if necessary if self.repo_path is None: self.repo_path = 'tmp-release-freertos' # Clean up any old work from previous runs if os.path.exists(self.repo_path): shutil.rmtree(self.repo_path) # Clone the target repo for creating the release autocommits remote_name = self.getRemoteEndpoint(self.repo_name) info('Downloading %s@%s to baseline auto-commits...' % (remote_name, commit), end='') self.local_repo = Repo.clone_from(remote_name, self.repo_path, progress=printDot, branch=self.branch) # In support of non-HEAD baselines self.local_repo.git.checkout(commit) print() def isValidManifestYML(self, path_yml): assert False, 'Unimplemented' def updateSubmodulePointers(self): ''' Reads the 'manifest.yml' file from the local FreeRTOS clone that is being used to stage the commits ''' info('Initializing first level of submodules...') self.local_repo.submodule_update(init=True, recursive=False) # Read YML file path_manifest = os.path.join(self.repo_path, 'manifest.yml') assert os.path.exists(path_manifest), 'Missing manifest.yml' with open(path_manifest, 'r') as fp: manifest_data = fp.read() yml = load(manifest_data, Loader=Loader) assert 'dependencies' in yml, 'Manifest YML parsing error' # Update all the submodules per yml logIndentPush() for dep in yml['dependencies']: assert 'version' in dep, 'Failed to parse submodule tag from manifest' assert 'repository' in dep and 'path' in dep['repository'], 'Failed to parse submodule path from manifest' submodule_path = dep['repository']['path'] submodule_tag = dep['version'] # Update the submodule to point to version noted in manifest file info('%-20s : %s' % (dep['name'], submodule_tag)) self.updateSubmodulePointer(os.path.join(self.repo_path, submodule_path), submodule_tag) logIndentPop() self.commitChanges(self.commit_msg_prefix + 'Bump submodules per manifest.yml for V%s' % self.version) def createReleaseZip(self): ''' At the moment, the only asset we upload is the ''' zip_name = 'FreeRTOSv%s' % self.version info('Packaging "%s"' % zip_name) logIndentPush() # This path name is retained in zip, so we don't name it 'tmp-*' but rather keep it consistent with previous # packaging rel_repo_path = zip_name # Clean up any old work from previous runs if os.path.exists(rel_repo_path): shutil.rmtree(rel_repo_path) # Download a fresh copy for packaging info('Downloading fresh copy of %s for packing...' % zip_name, end='') packaged_repo = Repo.clone_from(self.getRemoteEndpoint(self.repo_name), rel_repo_path, multi_options=['--depth=1', '-b%s' % self.tag, '--recurse-submodules'], progress=printDot, branch=self.branch) print() # Prune then zip package info('Pruning from release zip...', end='') files_pruned = prune_result_tree(rel_repo_path, FREERTOS_RELATIVE_FILE_EXCLUDES) print('...%d Files Removed.' % len(files_pruned)) info('Compressing "%s"...' % self.zip_path) with zipfile.ZipFile(self.zip_path, 'w', zipfile.ZIP_DEFLATED, compresslevel=9) as zip: for root, dirs, files in os.walk(rel_repo_path): for file in files: # For some strange reason, we have broken symlinks...avoid these file_path = os.path.join(root, file) if os.path.islink(file_path) and not os.path.exists(file_path): warning('Skipping over broken symlink "%s"' % file_path) else: zip.write(file_path) logIndentPop() def createGitRelease(self): ''' Creates/Overwrites release identified by target tag ''' if self.do_not_push: info('Skipping creating git release endpoint for "%s"...' % self.tag) else: # If this release already exists, delete it try: release_queried = self.repo.get_release(self.tag) info('Overwriting existing git release endpoint for "%s"...' % self.tag) release_queried.delete_release() except UnknownObjectException: info('Creating git release endpoint for "%s"...' % self.tag) # Create the new release endpoind at upload assets release = self.repo.create_git_release(tag = self.tag, name = 'FreeRTOSv%s' % (self.version), message = self.description, draft = False, prerelease = False) info('Uploading release asssets...') release.upload_asset(self.zip_path, name='FreeRTOSv%s.zip' % self.version, content_type='application/zip') def autoRelease(self): info('Auto-releasing FreeRTOS V%s' % self.version) self.updateFileHeaderVersions(['FreeRTOS Kernel V', 'FreeRTOS V', 'FreeRTOS Kernel ', 'FreeRTOS '], 'FreeRTOS V%s' % self.version) self.updateSubmodulePointers() # When baselining off a non-HEAD commit, main is left unchanged by tagging a detached HEAD, # applying the autocommits, tagging, and pushing the new tag data to remote. # However in the detached HEAD state we don't have a branch to push to, so we skip if self.commit == 'HEAD': self.pushLocalCommits() self.pushTag() self.createReleaseZip() self.createGitRelease() info('Core release done.') def configure_argparser(): parser = ArgumentParser(description='FreeRTOS Release tool') parser.add_argument('git_org', type=str, metavar='GITHUB_ORG', help='Git organization owner for FreeRTOS and FreeRTOS-Kernel. (i.e. "/FreeRTOS.git")') parser.add_argument('--new-core-version', default=None, required=False, help='FreeRTOS Standard Distribution Version to replace old version. (Ex. "FreeRTOS V202012.00")') parser.add_argument('--core-commit', default='HEAD', required=False, metavar='GITHUB_SHA', help='Github SHA to baseline autorelease') parser.add_argument('--rollback-core-version', default=None, required=False, help='Reset "main" to state prior to autorelease of given core version') parser.add_argument('--core-repo-path', type=str, default=None, required=False, help='Instead of downloading from git, use existing local repos for autocommits') parser.add_argument('--core-repo-branch', type=str, default='main', required=False, help='Branch of FreeRTOS hub repository to release.') parser.add_argument('--new-kernel-version', default=None, required=False, help='Reset "main" to just before the autorelease for the specified kernel version")') parser.add_argument('--new-kernel-main-br-version', default='', required=False, help='Set the version in task.h on the kernel main branch to the specified value.') parser.add_argument('--kernel-commit', default='HEAD', required=False, metavar='GITHUB_SHA', help='Github SHA to baseline autorelease') parser.add_argument('--rollback-kernel-version', default=None, required=False, help='Reset "main" to state prior to autorelease of the given kernel version') parser.add_argument('--kernel-repo-path', type=str, default=None, required=False, help='Instead of downloading from git, use existing local repos for autocommits') parser.add_argument('--kernel-repo-branch', type=str, default='main', required=False, help='Branch of FreeRTOS Kernel repository to release.') parser.add_argument('--use-git-ssh', default=False, action='store_true', help='Use SSH endpoints to interface git remotes, instead of HTTPS') parser.add_argument('--unit-test', action='store_true', default=False, help='Run unit tests.') parser.add_argument('--do-not-push', action='store_true', default=False, help='Do not push the changes but only make local commits.') return parser def main(): cmd = configure_argparser() args = cmd.parse_args() # Auth if not args.do_not_push: assert 'GITHUB_TOKEN' in os.environ, 'Set env{GITHUB_TOKEN} to an authorized git PAT' mGit = Github(os.environ.get('GITHUB_TOKEN')) # Unit tests if args.unit_test: return # Create Releases if args.new_kernel_version: info('Starting kernel release...') logIndentPush() rel_kernel = KernelRelease(mGit, args.new_kernel_version, args.kernel_commit, git_ssh=args.use_git_ssh, git_org=args.git_org, repo_path=args.kernel_repo_path, branch=args.kernel_repo_branch, main_br_version=args.new_kernel_main_br_version, do_not_push=args.do_not_push) rel_kernel.autoRelease() logIndentPop() if args.new_core_version: info('Starting core release...') logIndentPush() rel_freertos = FreertosRelease(mGit, args.new_core_version, args.core_commit, git_ssh=args.use_git_ssh, git_org=args.git_org, repo_path=args.core_repo_path, branch=args.core_repo_branch, do_not_push=args.do_not_push) rel_freertos.autoRelease() logIndentPop() # Undo autoreleases if args.rollback_kernel_version: info('Starting kernel rollback...') rel_kernel = KernelRelease(mGit, args.rollback_kernel_version, args.kernel_commit, git_ssh=args.use_git_ssh, git_org=args.git_org, repo_path=args.kernel_repo_path, branch=args.kernel_repo_branch, do_not_push=args.do_not_push) logIndentPush() rel_kernel.restorePriorToRelease() logIndentPop() if args.rollback_core_version: info('Starting core rollback...') logIndentPush() rel_freertos = FreertosRelease(mGit, args.rollback_core_version, args.core_commit, git_ssh=args.use_git_ssh, git_org=args.git_org, repo_path=args.core_repo_path, branch=args.core_repo_branch, do_not_push=args.do_not_push) rel_freertos.restorePriorToRelease() logIndentPop() info('Review script output for any unexpected behaviour.') info('Done.') if __name__ == '__main__': main()