From 0527a2a02a037ec6be354d10e2da564a62ee7d36 Mon Sep 17 00:00:00 2001 From: David Chalco <59750547+dachalco@users.noreply.github.com> Date: Sun, 20 Dec 2020 17:03:37 -0800 Subject: [PATCH] AutoRelease + Header Checker Upgrades (#482) * Initial version where all deps are pylibs * mechanism for undoing an autorelease * misc refactor touchups * +mechanism to baseline older commit into detached HEAD tag * decouple kernel check configs + misc refactor improvements * improved compatibility with git action * Get pushes working in git action with release * Fix header-check issue when same deletion occurs in parallel * Add help message in case check fails * Address PR feedback --- .github/scripts/check-header.py | 483 ------------------ .github/scripts/common/header_checker.py | 133 +++++ .github/scripts/core_checker.py | 313 ++++++++++++ .github/scripts/release-requirements.txt | 13 + .github/scripts/release.py | 449 ++++++++++++---- .github/scripts/versioning.py | 32 +- .github/workflows/auto-release.yml | 51 ++ .../{header-checks.yml => core-checks.yml} | 9 +- .github/workflows/release-packager.yml | 62 --- 9 files changed, 867 insertions(+), 678 deletions(-) delete mode 100755 .github/scripts/check-header.py create mode 100755 .github/scripts/common/header_checker.py create mode 100755 .github/scripts/core_checker.py create mode 100644 .github/scripts/release-requirements.txt create mode 100644 .github/workflows/auto-release.yml rename .github/workflows/{header-checks.yml => core-checks.yml} (81%) delete mode 100644 .github/workflows/release-packager.yml diff --git a/.github/scripts/check-header.py b/.github/scripts/check-header.py deleted file mode 100755 index f7273bbbd5..0000000000 --- a/.github/scripts/check-header.py +++ /dev/null @@ -1,483 +0,0 @@ -#!/usr/bin/env python3 - -import os, sys, re -from argparse import ArgumentParser -from difflib import unified_diff -from json import load - -def dprint(msg): - print('[DEBUG]: %s' % str(msg)) - -class HeaderChecker: - def __init__(self, header, padding=1000, ignored_files=[], ignored_ext=[], ignored_patterns=[]): - self.padding = padding - self.header = header - - self.ignorePatternList = ignored_patterns.copy() - self.ignoreFileList = ignored_files.copy() - self.ignoreExtList = ignored_ext.copy() - - def checkJSONList(self, path_json): - ''' - This is particularly useful when ingesting output from other programs, like git actions - ''' - assert os.path.exists(path_json), 'No such file: ' + path_json - - # Get list of files to check from JSON file - with open(path_json) as file_json: - file_checklist = load(file_json) - assert isinstance(file_checklist, list), 'Expected list for singular JSON List entry' - - # Accrue how how files fail the check - n_failed = 0 - for path_file in file_checklist: - assert isinstance(path_file, str), 'Unexpected JSON format for ' + path_json - n_failed += not self.isValidFile(path_file) - - return n_failed - - def isValidFile(self, path): - assert os.path.exists(path), 'No such file: ' + path - - # Skip any ignored files - if self.isIgnoredFile(path): - return True - - # Skip if entry is a directory. - if os.path.isdir(path): - print('Skipping valid file check on directory path: %s' % path) - return True - - # Don't need entire file. Read sufficienly large chunk of file that should contain the header - with open(path, encoding='utf-8', errors='ignore') as file: - chunk = file.read(len(''.join(self.header)) + self.padding) - lines = [('%s\n' % l) for l in chunk.strip().splitlines()][:len(self.header)] - if self.header == lines: - return True - else: - print('File Delta: %s' % path) - print(*unified_diff(lines[:len(self.header)], self.header)) - return False - - def ignoreExtension(self, *args): - for ext in args: - self.ignoreExtList.append(ext) - - def ignoreFile(self, *args): - for f in args: - self.ignoreFileList.append(f) - - def ignorePattern(self, *args): - for p in args: - self.ignorePatternList.append(re.compile(p)) - - def isIgnoredFile(self, path): - ''' - There are multiple ways a file can be ignored. This is a catch all - ''' - assert os.path.exists(path), 'No such file: ' + path - - - # Try simpler checks first - filename = os.path.split(path)[-1] - extension = os.path.splitext(filename)[-1] - if extension in self.ignoreExtList or filename in self.ignoreFileList: - return True - - # Then iterate against regex patterns. In future consider Trie - for pattern in self.ignorePatternList: - if pattern.match(path): - return True - - return False - - -def configArgParser(): - parser = ArgumentParser(description='FreeRTOS file header checker. We expect a consistent header across all ' - 'first party files. The header includes current version number, copyright, ' - 'and FreeRTOS license.') - - parser.add_argument('files_checked', - nargs = '+', - metavar = 'FILE_LIST', - help = 'Space separated list of files to check.') - - parser.add_argument('-k', '--kernel', - default = False, - action = 'store_true', - help = 'Compare with kernel file header. It has different versioning.') - - parser.add_argument('-j', '--json', - default = False, - action = 'store_true', - help = 'Treat arguments json files that store a list of files to check.') - return parser - -#-------------------------------------------------------------------------------------------------- -# CONFIG -#-------------------------------------------------------------------------------------------------- -FREERTOS_IGNORED_EXTENSIONS = [ - '.1', - '.ASM', - '.C', - '.DSW', - '.G_C', - '.H', - '.Hbp', - '.IDE', - '.LIB', - '.Opt', - '.PC', - '.PRM', - '.TXT', - '.URL', - '.UVL', - '.Uv2', - '.a', - '.ac', - '.am', - '.atsln', - '.atstart', - '.atsuo', - '.bash', - '.bat', - '.bbl', - '.bit', - '.board', - '.bsb', - '.bsdl', - '.bts', - '.ccxml', - '.cdkproj', - '.cdkws', - '.cfg', - '.cgp', - '.cmake', - '.cmd', - '.config', - '.cpp', - '.cproj', - '.crun', - '.css', - '.csv', - '.custom_argvars', - '.cxx', - '.cydwr', - '.cyprj', - '.cysch', - '.dat', - '.datas', - '.db', - '.dbgdt', - '.dep', - '.dni', - '.dnx', - '.doc', - '.dox', - '.doxygen', - '.ds', - '.dsk', - '.dtd', - '.dts', - '.elf', - '.env_conf', - '.ewd', - '.ewp', - '.ewt', - '.eww', - '.exe', - '.filters', - '.flash', - '.fmt', - '.ftl', - '.gdb', - '.gif', - '.gise', - '.gld', - '.gpdsc', - '.gui', - '.h_from_toolchain', - '.hdf', - '.hdp', - '.hex', - '.hist', - '.history', - '.hsf', - '.htm', - '.html', - '.hwc', - '.hwl', - '.hwp', - '.hws', - '.hzp', - '.hzs', - '.i', - '.icf', - '.ide', - '.idx', - '.in', - '.inc', - '.include', - '.index', - '.inf', - '.ini', - '.init', - '.ipcf', - '.ise', - '.jlink', - '.json', - '.la', - '.launch', - '.lcf', - '.lds', - '.lib', - '.lk1', - '.lkr', - '.lm', - '.lo', - '.lock', - '.lsl', - '.lst', - '.m4', - '.mac', - '.make', - '.map', - '.mbt', - '.mcp', - '.mcpar', - '.mcs', - '.mcw', - '.md', - '.mdm', - '.mem', - '.mhs', - '.mk', - '.mk1', - '.mmi', - '.mrt', - '.mss', - '.mtpj', - '.nav', - '.ntrc_log', - '.opa', - '.opb', - '.opc', - '.opl', - '.opt', - '.opv', - '.out', - '.pack', - '.par', - '.patch', - '.pbd', - '.pdsc', - '.pe', - '.pem', - '.pgs', - '.pl', - '.plg', - '.png', - '.prc', - '.pref', - '.prefs', - '.prj', - '.properties', - '.ps1', - '.ptf', - '.r79', - '.rapp', - '.rc', - '.reggroups', - '.reglist', - '.resc', - '.resources', - '.rom', - '.rprj', - '.s79', - '.s82', - '.s90', - '.sc', - '.scf', - '.scfg', - '.script', - '.sct', - '.scvd', - '.session', - '.sfr', - '.sh', - '.shtml', - '.sig', - '.sln', - '.spec', - '.stf', - '.stg', - '.suo', - '.sup', - '.svg', - '.tags', - '.tcl', - '.tdt', - '.template', - '.tgt', - '.tps', - '.tra', - '.tree', - '.tws', - '.txt', - '.ucf', - '.url', - '.user', - '.ut', - '.uvmpw', - '.uvopt', - '.uvoptx', - '.uvproj', - '.uvprojx', - '.vcproj', - '.vcxproj', - '.version', - '.webserver', - '.wpj', - '.wsdt', - '.wsp', - '.wspos', - '.wsx', - '.x', - '.xbcd', - '.xcl', - '.xise', - '.xml', - '.xmp', - '.xmsgs', - '.xsl', - '.yml', - '.md', - '.zip' -] - -FREERTOS_IGNORED_PATTERNS = [ - r'.*\.git.*', - r'.*mbedtls_config\.h.*', - r'.*mbedtls_config\.h.*', - r'.*CMSIS.*', - r'.*/makefile', - r'.*/Makefile', - r'.*/trcConfig\.h.*', - r'.*/trcConfig\.c.*', - r'.*/trcSnapshotConfig\.h.*', -] - -FREERTOS_HEADER = [ - '/*\n', - ' * FreeRTOS V202012.00\n', - ' * Copyright (C) 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n', - ' *\n', - ' * Permission is hereby granted, free of charge, to any person obtaining a copy of\n', - ' * this software and associated documentation files (the "Software"), to deal in\n', - ' * the Software without restriction, including without limitation the rights to\n', - ' * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\n', - ' * the Software, and to permit persons to whom the Software is furnished to do so,\n', - ' * subject to the following conditions:\n', - ' *\n', - ' * The above copyright notice and this permission notice shall be included in all\n', - ' * copies or substantial portions of the Software.\n', - ' *\n', - ' * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n', - ' * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\n', - ' * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\n', - ' * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\n', - ' * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n', - ' * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n', - ' *\n', - ' * https://www.FreeRTOS.org\n', - ' * https://github.com/FreeRTOS\n', - ' *\n', - ' */\n', -] - -KERNEL_IGNORED_EXTENSIONS = [ - '.yml', - '.css', - '.idx', - '.md', - '.url', - '.sty', - '.0-rc2', - '.s82', - '.js', - '.out', - '.pack', - '.2', - '.1-kernel-only', - '.0-kernel-only', - '.0-rc1', - '.readme', - '.tex', - '.png', - '.bat', - '.sh' -] - -KERNEL_IGNORED_PATTERNS = [r'.*\.git.*'] - -KERNEL_HEADER = [ - '/*\n', - ' * FreeRTOS Kernel V10.4.2\n', - ' * Copyright (C) 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n', - ' *\n', - ' * Permission is hereby granted, free of charge, to any person obtaining a copy of\n', - ' * this software and associated documentation files (the "Software"), to deal in\n', - ' * the Software without restriction, including without limitation the rights to\n', - ' * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\n', - ' * the Software, and to permit persons to whom the Software is furnished to do so,\n', - ' * subject to the following conditions:\n', - ' *\n', - ' * The above copyright notice and this permission notice shall be included in all\n', - ' * copies or substantial portions of the Software.\n', - ' *\n', - ' * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n', - ' * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\n', - ' * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\n', - ' * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\n', - ' * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n', - ' * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n', - ' *\n', - ' * https://www.FreeRTOS.org\n', - ' * https://github.com/FreeRTOS\n', - ' *\n', - ' */\n', -] - - -#-------------------------------------------------------------------------------------------------- -# MAIN -#-------------------------------------------------------------------------------------------------- -def main(): - parser = configArgParser() - args = parser.parse_args() - - # Configure checks - if args.kernel: - checker = HeaderChecker(KERNEL_HEADER) - checker.ignoreExtension(*KERNEL_IGNORED_EXTENSIONS) - checker.ignorePattern(*KERNEL_IGNORED_PATTERNS) - else: - checker = HeaderChecker(FREERTOS_HEADER) - checker.ignoreExtension(*FREERTOS_IGNORED_EXTENSIONS) - checker.ignorePattern(*FREERTOS_IGNORED_PATTERNS) - - checker.ignoreFile(os.path.split(__file__)[-1]) - - # Check all input files - print() - n_failed = 0 - for path in args.files_checked: - if args.json: - n_failed += checker.checkJSONList(path) - else: - n_failed += not checker.isValidFile(path) - - return n_failed - -if __name__ == '__main__': - exit(main()) diff --git a/.github/scripts/common/header_checker.py b/.github/scripts/common/header_checker.py new file mode 100755 index 0000000000..607568321c --- /dev/null +++ b/.github/scripts/common/header_checker.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 + +import os, sys, re +from argparse import ArgumentParser +from difflib import unified_diff +from json import load + +def dprint(msg): + print('[DEBUG]: %s' % str(msg)) + +class HeaderChecker: + def __init__(self, header, padding=1000, ignored_files=[], ignored_ext=[], ignored_patterns=[]): + self.padding = padding + self.header = header + + self.ignorePatternList = ignored_patterns.copy() + self.ignoreFileList = ignored_files.copy() + self.ignoreExtList = ignored_ext.copy() + + def checkJSONList(self, path_json): + ''' + This is particularly useful when ingesting output from other programs, like git actions + ''' + assert os.path.exists(path_json), 'No such file: ' + path_json + + # Get list of files to check from JSON file + with open(path_json) as file_json: + file_checklist = load(file_json) + assert isinstance(file_checklist, list), 'Expected list for singular JSON List entry' + + # Accrue how how files fail the check + n_failed = 0 + for path_file in file_checklist: + assert isinstance(path_file, str), 'Unexpected JSON format for ' + path_json + if os.path.exists(path_file) and not self.isValidFile(path_file): + n_failed += 1 + + return n_failed + + def isValidFile(self, path): + assert os.path.exists(path), 'No such file: ' + path + print('-------------------------------------------------------------------------------------') + + print('Checking file: %s...' % path, end='') + + if self.isIgnoredFile(path) or os.path.isdir(path): + print('SKIP') + print('-------------------------------------------------------------------------------------') + return True + + # Don't need entire file. Read sufficiently large chunk of file that should contain the header + with open(path, encoding='utf-8', errors='ignore') as file: + chunk = file.read(len(''.join(self.header)) + self.padding) + lines = [('%s\n' % l) for l in chunk.strip().splitlines()][:len(self.header)] + if self.header == lines: + print('PASS') + print('-------------------------------------------------------------------------------------') + return True + else: + print('FAIL') + print('File Delta: %s' % path) + print(*unified_diff(lines[:len(self.header)], self.header)) + print('-------------------------------------------------------------------------------------') + return False + + def ignoreExtension(self, *args): + for ext in args: + self.ignoreExtList.append(ext) + + def ignoreFile(self, *args): + for f in args: + self.ignoreFileList.append(f) + + def ignorePattern(self, *args): + for p in args: + self.ignorePatternList.append(re.compile(p)) + + def isIgnoredFile(self, path): + ''' + There are multiple ways a file can be ignored. This is a catch all + ''' + assert os.path.exists(path), 'No such file: ' + path + + + # Try simpler checks first + filename = os.path.split(path)[-1] + extension = os.path.splitext(filename)[-1] + if extension in self.ignoreExtList or filename in self.ignoreFileList: + return True + + # Then iterate against regex patterns. In future consider Trie + for pattern in self.ignorePatternList: + if pattern.match(path): + return True + + return False + + def showHelp(self, path_config): + print('\n\n' + "Please fix all highlighted diffs or add exceptions to '%s' as necessary.\n" + "Include your changes to '%s' in your PR. Git PR checks source this file from your PR.\n" + "\n" + "The FreeRTOS Header Check ensures all files that contain FreeRTOS Headers are up to date\n" + "with the latest version, copyright, and licensing info." + "\n\n" % (path_config, path_config)) + + @staticmethod + def configArgParser(): + parser = ArgumentParser(description='FreeRTOS file header checker. We expect a consistent header across all ' + 'first party files. The header includes current version number, copyright, ' + 'and FreeRTOS license.') + + parser.add_argument('files_checked', + nargs = '+', + metavar = 'FILE_LIST', + help = 'Space separated list of files to check.') + + parser.add_argument('-j', '--json', + default = False, + action = 'store_true', + help = 'Treat arguments json files that store a list of files to check.') + return parser + + def processArgs(self, args): + n_failed = 0 + if args.json: + for path in args.files_checked: + n_failed += self.checkJSONList(path) + else: + for path in args.files_checked: + n_failed += not self.isValidFile(path) + + return n_failed diff --git a/.github/scripts/core_checker.py b/.github/scripts/core_checker.py new file mode 100755 index 0000000000..05d83ce0df --- /dev/null +++ b/.github/scripts/core_checker.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +# python >= 3.4 + +import os +from common.header_checker import HeaderChecker + +#-------------------------------------------------------------------------------------------------- +# CONFIG +#-------------------------------------------------------------------------------------------------- +FREERTOS_IGNORED_EXTENSIONS = [ + '.1', + '.ASM', + '.C', + '.DSW', + '.G_C', + '.H', + '.Hbp', + '.IDE', + '.LIB', + '.Opt', + '.PC', + '.PRM', + '.TXT', + '.URL', + '.UVL', + '.Uv2', + '.a', + '.ac', + '.am', + '.atsln', + '.atstart', + '.atsuo', + '.bash', + '.bat', + '.bbl', + '.bit', + '.board', + '.bsb', + '.bsdl', + '.bts', + '.ccxml', + '.cdkproj', + '.cdkws', + '.cfg', + '.cgp', + '.cmake', + '.cmd', + '.config', + '.cpp', + '.cproj', + '.crun', + '.css', + '.csv', + '.custom_argvars', + '.cxx', + '.cydwr', + '.cyprj', + '.cysch', + '.dat', + '.datas', + '.db', + '.dbgdt', + '.dep', + '.dni', + '.dnx', + '.doc', + '.dox', + '.doxygen', + '.ds', + '.dsk', + '.dtd', + '.dts', + '.elf', + '.env_conf', + '.ewd', + '.ewp', + '.ewt', + '.eww', + '.exe', + '.filters', + '.flash', + '.fmt', + '.ftl', + '.gdb', + '.gif', + '.gise', + '.gld', + '.gpdsc', + '.gui', + '.h_from_toolchain', + '.hdf', + '.hdp', + '.hex', + '.hist', + '.history', + '.hsf', + '.htm', + '.html', + '.hwc', + '.hwl', + '.hwp', + '.hws', + '.hzp', + '.hzs', + '.i', + '.icf', + '.ide', + '.idx', + '.in', + '.inc', + '.include', + '.index', + '.inf', + '.ini', + '.init', + '.ipcf', + '.ise', + '.jlink', + '.json', + '.la', + '.launch', + '.lcf', + '.lds', + '.lib', + '.lk1', + '.lkr', + '.lm', + '.lo', + '.lock', + '.lsl', + '.lst', + '.m4', + '.mac', + '.make', + '.map', + '.mbt', + '.mcp', + '.mcpar', + '.mcs', + '.mcw', + '.md', + '.mdm', + '.mem', + '.mhs', + '.mk', + '.mk1', + '.mmi', + '.mrt', + '.mss', + '.mtpj', + '.nav', + '.ntrc_log', + '.opa', + '.opb', + '.opc', + '.opl', + '.opt', + '.opv', + '.out', + '.pack', + '.par', + '.patch', + '.pbd', + '.pdsc', + '.pe', + '.pem', + '.pgs', + '.pl', + '.plg', + '.png', + '.prc', + '.pref', + '.prefs', + '.prj', + '.properties', + '.ps1', + '.ptf', + '.r79', + '.rapp', + '.rc', + '.reggroups', + '.reglist', + '.resc', + '.resources', + '.rom', + '.rprj', + '.s79', + '.s82', + '.s90', + '.sc', + '.scf', + '.scfg', + '.script', + '.sct', + '.scvd', + '.session', + '.sfr', + '.sh', + '.shtml', + '.sig', + '.sln', + '.spec', + '.stf', + '.stg', + '.suo', + '.sup', + '.svg', + '.tags', + '.tcl', + '.tdt', + '.template', + '.tgt', + '.tps', + '.tra', + '.tree', + '.tws', + '.txt', + '.ucf', + '.url', + '.user', + '.ut', + '.uvmpw', + '.uvopt', + '.uvoptx', + '.uvproj', + '.uvprojx', + '.vcproj', + '.vcxproj', + '.version', + '.webserver', + '.wpj', + '.wsdt', + '.wsp', + '.wspos', + '.wsx', + '.x', + '.xbcd', + '.xcl', + '.xise', + '.xml', + '.xmp', + '.xmsgs', + '.xsl', + '.yml', + '.md', + '.zip' +] + +FREERTOS_IGNORED_PATTERNS = [ + r'.*\.git.*', + r'.*mbedtls_config\.h.*', + r'.*mbedtls_config\.h.*', + r'.*CMSIS.*', + r'.*/makefile', + r'.*/Makefile', + r'.*/trcConfig\.h.*', + r'.*/trcConfig\.c.*', + r'.*/trcSnapshotConfig\.h.*', +] + +FREERTOS_IGNORED_FILES = [ + 'fyi-another-way-to-ignore-file.txt', + 'mbedtls_config.h' +] + +FREERTOS_HEADER = [ + '/*\n', + ' * FreeRTOS V202012.00\n', + ' * Copyright (C) 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n', + ' *\n', + ' * Permission is hereby granted, free of charge, to any person obtaining a copy of\n', + ' * this software and associated documentation files (the "Software"), to deal in\n', + ' * the Software without restriction, including without limitation the rights to\n', + ' * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\n', + ' * the Software, and to permit persons to whom the Software is furnished to do so,\n', + ' * subject to the following conditions:\n', + ' *\n', + ' * The above copyright notice and this permission notice shall be included in all\n', + ' * copies or substantial portions of the Software.\n', + ' *\n', + ' * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n', + ' * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\n', + ' * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\n', + ' * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\n', + ' * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n', + ' * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n', + ' *\n', + ' * https://www.FreeRTOS.org\n', + ' * https://github.com/FreeRTOS\n', + ' *\n', + ' */\n', +] + +def main(): + parser = HeaderChecker.configArgParser() + args = parser.parse_args() + + # Configure the checks then run + checker = HeaderChecker(FREERTOS_HEADER) + checker.ignoreExtension(*FREERTOS_IGNORED_EXTENSIONS) + checker.ignorePattern(*FREERTOS_IGNORED_PATTERNS) + checker.ignoreFile(*FREERTOS_IGNORED_FILES) + checker.ignoreFile(os.path.split(__file__)[-1]) + + rc = checker.processArgs(args) + if rc: + checker.showHelp(__file__) + + return rc + +if __name__ == '__main__': + exit(main()) + diff --git a/.github/scripts/release-requirements.txt b/.github/scripts/release-requirements.txt new file mode 100644 index 0000000000..6cd9244535 --- /dev/null +++ b/.github/scripts/release-requirements.txt @@ -0,0 +1,13 @@ +certifi>=2020.12.5 +chardet>=3.0.4 +Deprecated>=1.2.10 +gitdb>=4.0.5 +GitPython>=3.1.11 +idna>=2.10 +PyGithub>=1.54 +PyJWT>=1.7.1 +PyYAML>=5.3.1 +requests>=2.24.0 +smmap>=3.0.4 +urllib3>=1.25.11 +wrapt>=1.12.1 diff --git a/.github/scripts/release.py b/.github/scripts/release.py index 488ca0b530..4716c91488 100755 --- a/.github/scripts/release.py +++ b/.github/scripts/release.py @@ -7,58 +7,75 @@ except ImportError: from yaml import Loader, Dumper from argparse import ArgumentParser +# For interfacing Git REST API import re import datetime from github import Github from github.GithubException import * from github.InputGitAuthor import InputGitAuthor +# Local interfacing of repo +from git import Repo +from git import PushInfo + +import zipfile + from versioning import update_version_number_in_freertos_component from versioning import update_freertos_version_macros -from packager import download_git_tree -from packager import update_submodule_pointer -from packager import commit_git_tree_changes -from packager import push_git_tree_changes -from packager import create_package +from packager import prune_result_tree from packager import RELATIVE_FILE_EXCLUDES as FREERTOS_RELATIVE_FILE_EXCLUDES # PyGithub Git - https://github.com/PyGithub/PyGithub # PyGithub Docs - https://pygithub.readthedocs.io/en/latest/github_objects # REST API used by PyGithub - https://developer.github.com/v3/ -def info(msg, indent_level=0): - print('%s[INFO]: %s' % (' ' * indent_level, str(msg))) +indent_level = 0 + +def logIndentPush(): + global indent_level + indent_level += 4 + +def logIndentPop(): + global indent_level + indent_level -= 4 + + if indent_level < 0: + indent_level = 0 -def warning(msg, indent_level=0): - print('%s[WARNING]: %s' % (' ' * indent_level, str(msg))) +def info(msg, end='\n'): + print('[INFO]: %s%s' % (' ' * indent_level, str(msg)), end=end, flush=True) -def error(msg, indent_level=0): - print('%s[ERROR]: %s' % (' ' * indent_level, str(msg))) +def warning(msg): + print('[WARNING]: %s%s' % (' ' * indent_level, str(msg)), flush=True) -def debug(msg, indent_level=0): - print('%s[DEBUG]: %s' % (' ' * indent_level, str(msg))) +def error(msg): + print('[ERROR]: %s%s' % (' ' * indent_level, str(msg)), flush=True) + +def debug(msg): + print('[DEBUG]: %s%s' % (' ' * indent_level, str(msg)), flush=True) + +# Callback for progress updates. For long spanning gitpython commands +def printDot(op_code, cur_count, max_count=None, message=''): + if max_count == None or cur_count == max_count: + print('.', end='') class BaseRelease: - def __init__(self, mGit, version, commit, git_ssh=False, git_org='FreeRTOS'): + def __init__(self, mGit, version, commit='HEAD', git_ssh=False, git_org='FreeRTOS', repo_path=None): self.version = version self.tag_msg = 'Autocreated by FreeRTOS Git Tools.' self.commit = commit self.git_ssh = git_ssh self.git_org = git_org - self.repo_path = None + self.repo_path = repo_path + self.local_repo = None self.commit_msg_prefix = '[AUTO][RELEASE]: ' self.description = '' - self.mGit = mGit # Save a handle to the authed git session - def updateFileHeaderVersions(self): - ''' - Updates for all FreeRTOS/FreeRTOS files, not including submodules, to have their file header - versions updated to match this release version. It creates the release tag and stores these updates there, - at a detached commit (akin to a branch). - ''' - assert False, 'Implement me' + if self.repo_path: + info('Sourcing "%s" to make local commits' % self.repo_path) + self.local_repo = Repo(self.repo_path) def CheckRelease(self): ''' @@ -75,6 +92,13 @@ class BaseRelease: return False + def commitChanges(self, msg): + assert self.local_repo != None, 'Failed to commit. Git repo uninitialized.' + + info('Committing: "%s"' % msg) + self.local_repo.git.add(update=True) + commit = self.local_repo.index.commit(msg) + def getRemoteEndpoint(self, repo_name): if self.git_ssh: return 'git@github.com:%s.git' % repo_name @@ -86,60 +110,148 @@ class BaseRelease: for r in releases: print(r) - def pushAutoCommits(self): - rc = push_git_tree_changes(self.repo_path, tag=self.tag, force_tag=True) - assert rc == 0, 'Failed to upload git tree changes' + def pushLocalCommits(self, force=False): + info('Pushing local commits...') + push_infos = self.local_repo.remote('origin').push(force=force) + + # Check for any errors + for push_info in push_infos: + assert 0 == push_info.flags & PushInfo.ERROR, 'Failed to push changes to ' + str(push_info) + + def pushTag(self): + # Overwrite existing tags + info('Pushing tag "%s"' % self.tag) + tag_info = self.local_repo.create_tag(self.tag, message=self.tag_msg, force=True) + self.local_repo.git.push(tags=True, force=True) + + def deleteTag(self): + # Remove from remote + if self.tag in self.local_repo.tags: + info('Deleting tag "%s"' % self.tag) + self.local_repo.remote('origin').push(':%s' % self.tag) + else: + info('A tag does not exists for "%s". No need to delete.' % self.tag) + + def updateSubmodulePointer(self, rel_path, ref): + submodule = Repo(rel_path) + submodule.remote('origin').fetch() + submodule.git.checkout(ref) + + def updateFileHeaderVersions(self, old_version_substrings, new_version_string): + info('Updating file header versions for "%s"...' % self.version, end='') + n_updated = 0 + n_updated += update_version_number_in_freertos_component(self.repo_path, + '.', + old_version_substrings, + new_version_string, + exclude_hidden=True) + + n_updated += update_version_number_in_freertos_component(os.path.join('.github', 'scripts'), + self.repo_path, + old_version_substrings, + new_version_string, + exclude_hidden=False) + + print('...%d Files updated.' % n_updated) + + self.commitChanges(self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version) + + def deleteGitRelease(self): + info('Deleting git release endpoint for "%s"' % self.tag) + + try: + self.repo.get_release(self.tag).delete_release() + except UnknownObjectException: + info('A release endpoint does not exist for "%s". No need to erase.' % self.tag) + except: + assert False, 'Encountered error while trying to delete git release endpoint' + + def rollbackAutoCommits(self, n_autocommits=2, n_search=25): + info('Rolling back "%s" autocommits' % self.tag) + + if self.tag not in self.local_repo.tags: + error('Could not find a SHA to rollback to for tag "%s"' % self.tag) + return False + + # Search for auto release SHAs that match the release tag SHA + tag_commit = self.local_repo.tag('refs/tags/%s' % self.tag).commit + prior_commit = self.local_repo.commit(tag_commit.hexsha + '~%d' % n_autocommits) + n_commits_searched = 0 + for commit in self.local_repo.iter_commits(): + if n_commits_searched > n_search: + error('Exhaustively searched but could not find tag commit to rollback') + return False + + if (self.commit_msg_prefix in commit.message + and commit.hexsha == tag_commit.hexsha + and self.version in commit.message): + + info('Found matching tag commit %s. Reverting to prior commit %s' + % (tag_commit.hexsha, prior_commit.hexsha)) + + # Found the commit prior to this autorelease. Revert back to it then push + self.local_repo.git.reset(prior_commit.hexsha, hard=True) + self.pushLocalCommits(force=True) + return True + + n_commits_searched += 1 + + return False + + def restorePriorToRelease(self): + info('Restoring "master" to just before autorelease:%s' % self.version) + + self.deleteGitRelease() + self.rollbackAutoCommits() + self.deleteTag() + self.pushLocalCommits(force=True) + class KernelRelease(BaseRelease): - def __init__(self, mGit, version, commit, git_ssh=False, git_org='FreeRTOS'): - super().__init__(mGit, version, commit, git_ssh=git_ssh, git_org=git_org) + def __init__(self, mGit, version, commit='HEAD', git_ssh=False, git_org='FreeRTOS', repo_path=None): + super().__init__(mGit, version, commit=commit, git_ssh=git_ssh, git_org=git_org, repo_path=repo_path) self.repo_name = '%s/FreeRTOS-Kernel' % self.git_org self.repo = mGit.get_repo(self.repo_name) self.tag = 'V%s' % version - # Download a local git repo for pushing commits - remote_name = self.getRemoteEndpoint(self.repo_name) - self.repo_path = 'tmp-release-freertos-kernel' + # Parent ctor configures local_repo if caller chooses to source local repo from repo_path. + if self.repo_path is None: + self.repo_path = 'tmp-release-freertos-kernel' + if os.path.exists(self.repo_path): + shutil.rmtree(self.repo_path) - # Clean up any old work from previous runs - if os.path.exists(self.repo_path): - shutil.rmtree(self.repo_path) + # Clone the target repo for creating the release autocommits + remote_name = self.getRemoteEndpoint(self.repo_name) + info('Downloading %s@%s to baseline auto-commits...' % (remote_name, commit), end='') + self.local_repo = Repo.clone_from(remote_name, self.repo_path, progress=printDot) - # Download master:HEAD. Update its file header versions and kernel macros - self.repo_path = download_git_tree(remote_name, '.', self.repo_path, 'master', 'HEAD') - assert self.repo_path != None, 'Failed to download git tree' + # In case user gave non-HEAD commit to baseline + self.local_repo.git.checkout(commit) + + print() - def updateFileHeaderVersions(self): - ''' - Adds changes for two commits - 1.) Updates to file headers - 2.) Update to task.h macros - Then tags commit #2 with the new tag version. Notes this will overwrite a tag it already exists - Finally pushes all these changes - ''' - target_version_prefixes = ['FreeRTOS Kernel V'] - update_version_number_in_freertos_component(self.repo_path, '.', target_version_prefixes, 'FreeRTOS Kernel V%s' % self.version) - commit_git_tree_changes(self.repo_path, commit_message=self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version) def updateVersionMacros(self): + info('Updating version macros in task.h for "%s"' % self.version) + (major, minor, build) = self.version.split('.') update_freertos_version_macros(os.path.join(self.repo_path, 'include', 'task.h'), major, minor, build) - commit_git_tree_changes(self.repo_path, commit_message=self.commit_msg_prefix + 'Bump task.h version macros to "%s"' % self.version) + + self.commitChanges(self.commit_msg_prefix + 'Bump task.h version macros to "%s"' % self.version) def createGitRelease(self): ''' Creates/Overwrites release identified by target tag ''' - # If this release already exists, delete it try: release_queried = self.repo.get_release(self.tag) - info('Deleting existing release "%s"...' % self.tag) + info('Overwriting existing git release endpoint for "%s"...' % self.tag) release_queried.delete_release() except UnknownObjectException: - info('Creating release/tag "%s"...' % self.tag) + info('Creating git release endpoint for "%s"...' % self.tag) # Create the new release endpoint at upload assets release = self.repo.create_git_release(tag = self.tag, @@ -148,43 +260,73 @@ class KernelRelease(BaseRelease): draft = False, prerelease = False) + def autoRelease(self): + info('Auto-releasing FreeRTOS Kernel V%s' % self.version) + + self.updateFileHeaderVersions(['FreeRTOS Kernel V'], 'FreeRTOS Kernel V%s' % self.version) + self.updateVersionMacros() + + # When baselining off a non-HEAD commit, master is left unchanged by tagging a detached HEAD, + # applying the autocommits, tagging, and pushing the new tag data to remote. + # However in the detached HEAD state we don't have a branch to push to, so we skip + if self.commit == 'HEAD': + self.pushLocalCommits() + + self.pushTag() + self.createGitRelease() + + info('Kernel release done.') + + + class FreertosRelease(BaseRelease): - def __init__(self, mGit, version, commit, git_ssh=False, git_org='FreeRTOS'): - super().__init__(mGit, version, commit, git_ssh=git_ssh, git_org=git_org) + def __init__(self, mGit, version, commit, git_ssh=False, git_org='FreeRTOS', repo_path=None): + super().__init__(mGit, version, commit, git_ssh=git_ssh, git_org=git_org, repo_path=repo_path) self.repo_name = '%s/FreeRTOS' % self.git_org self.repo = mGit.get_repo(self.repo_name) self.tag = self.version self.description = 'Contains source code and example projects for the FreeRTOS Kernel and FreeRTOS+ libraries.' - self.zip = None + self.zip_path = 'FreeRTOSv%s.zip' % self.version - remote_name = self.getRemoteEndpoint(self.repo_name) - self.repo_path = 'tmp-release-freertos' + # Download a fresh copy of local repo for making autocommits, if necessary + if self.repo_path is None: + self.repo_path = 'tmp-release-freertos' - # Clean up any old work from previous runs - if os.path.exists(self.repo_path): - shutil.rmtree(self.repo_path) + # Clean up any old work from previous runs + if os.path.exists(self.repo_path): + shutil.rmtree(self.repo_path) + + # Clone the target repo for creating the release autocommits + remote_name = self.getRemoteEndpoint(self.repo_name) + info('Downloading %s@%s to baseline auto-commits...' % (remote_name, commit), end='') + self.local_repo = Repo.clone_from(remote_name, self.repo_path, progress=printDot) - # Download master:HEAD. Update its file header versions and kernel submodule pointer - self.repo_path = download_git_tree(remote_name, '.', self.repo_path, 'master', 'HEAD') - assert self.repo_path != None, 'Failed to download git tree' + # In support of non-HEAD baselines + self.local_repo.git.checkout(commit) + print() - def updateFileHeaderVersions(self): - target_version_substrings = ['FreeRTOS Kernel V', 'FreeRTOS V'] - update_version_number_in_freertos_component(self.repo_path, '.', target_version_substrings, 'FreeRTOS V%s' % self.version) - commit_git_tree_changes(self.repo_path, commit_message=self.commit_msg_prefix + 'Bump file header version to "%s"' % self.version) + def isValidManifestYML(self, path_yml): + assert False, 'Unimplemented' def updateSubmodulePointers(self): ''' Reads the 'manifest.yml' file from the local FreeRTOS clone that is being used to stage the commits ''' + + info('Initializing first level of submodules...') + self.local_repo.submodule_update(init=True, recursive=False) + + # Read YML file path_manifest = os.path.join(self.repo_path, 'manifest.yml') assert os.path.exists(path_manifest), 'Missing manifest.yml' - with open(path_manifest, 'r') as fp: manifest_data = fp.read() yml = load(manifest_data, Loader=Loader) assert 'dependencies' in yml, 'Manifest YML parsing error' + + # Update all the submodules per yml + logIndentPush() for dep in yml['dependencies']: assert 'version' in dep, 'Failed to parse submodule tag from manifest' assert 'repository' in dep and 'path' in dep['repository'], 'Failed to parse submodule path from manifest' @@ -192,52 +334,66 @@ class FreertosRelease(BaseRelease): submodule_tag = dep['version'] # Update the submodule to point to version noted in manifest file - update_submodule_pointer(self.repo_path, submodule_path, submodule_tag) + info('%-20s : %s' % (dep['name'], submodule_tag)) + self.updateSubmodulePointer(os.path.join(self.repo_path, submodule_path), submodule_tag) + logIndentPop() - commit_git_tree_changes(self.repo_path, commit_message=self.commit_msg_prefix - + 'Bump submodules per manifest.yml for V%s' % self.version) + self.commitChanges(self.commit_msg_prefix + 'Bump submodules per manifest.yml for V%s' % self.version) def createReleaseZip(self): ''' At the moment, the only asset we upload is the ''' - remote_name = self.getRemoteEndpoint(self.repo_name) + zip_name = 'FreeRTOSv%s' % self.version + info('Packaging "%s"' % zip_name) + logIndentPush() # This path name is retained in zip, so we don't name it 'tmp-*' but rather keep it consistent with previous # packaging - repo_name = 'FreeRTOSv%s' % self.version - zip_root_path = repo_name - rel_repo_path = os.path.join(zip_root_path, repo_name) + rel_repo_path = zip_name # Clean up any old work from previous runs - if os.path.exists(zip_root_path): - shutil.rmtree(zip_root_path) - - # To keep consistent with previous packages - os.mkdir(zip_root_path) - - # Download master:HEAD. Update its file header versions and kernel submodule pointer - repo_path = download_git_tree(remote_name, '.', rel_repo_path, 'master', self.tag, recurse=True) - assert repo_path != None, 'Failed to download git tree' - - self.zip = create_package(zip_root_path, - rel_repo_path, - 'FreeRTOSv%s' % self.version, - exclude_files=FREERTOS_RELATIVE_FILE_EXCLUDES) + if os.path.exists(rel_repo_path): + shutil.rmtree(rel_repo_path) + + # Download a fresh copy for packaging + info('Downloading fresh copy of %s for packing...' % zip_name, end='') + packaged_repo = Repo.clone_from(self.getRemoteEndpoint(self.repo_name), + rel_repo_path, + multi_options=['--depth=1', '-b%s' % self.tag, '--recurse-submodules'], + progress=printDot) + print() + + # Prune then zip package + info('Pruning from release zip...', end='') + files_pruned = prune_result_tree(rel_repo_path, FREERTOS_RELATIVE_FILE_EXCLUDES) + print('...%d Files Removed.' % len(files_pruned)) + + info('Compressing "%s"...' % self.zip_path) + with zipfile.ZipFile(self.zip_path, 'w', zipfile.ZIP_DEFLATED, compresslevel=9) as zip: + for root, dirs, files in os.walk(rel_repo_path): + for file in files: + # For some strange reason, we have broken symlinks...avoid these + file_path = os.path.join(root, file) + if os.path.islink(file_path) and not os.path.exists(file_path): + warning('Skipping over broken symlink "%s"' % file_path) + else: + zip.write(file_path) + + logIndentPop() def createGitRelease(self): ''' Creates/Overwrites release identified by target tag ''' - # If this release already exists, delete it try: release_queried = self.repo.get_release(self.tag) - info('Deleting existing release "%s"...' % self.tag) + info('Overwriting existing git release endpoint for "%s"...' % self.tag) release_queried.delete_release() except UnknownObjectException: - info('Creating release/tag "%s"...' % self.tag) + info('Creating git release endpoint for "%s"...' % self.tag) # Create the new release endpoind at upload assets release = self.repo.create_git_release(tag = self.tag, @@ -246,25 +402,77 @@ class FreertosRelease(BaseRelease): draft = False, prerelease = False) - release.upload_asset(self.zip, name='FreeRTOSv%s.zip' % self.version, content_type='application/zip') + info('Uploading release asssets...') + release.upload_asset(self.zip_path, name='FreeRTOSv%s.zip' % self.version, content_type='application/zip') + + def autoRelease(self): + info('Auto-releasing FreeRTOS V%s' % self.version) + + self.updateFileHeaderVersions(['FreeRTOS Kernel V', 'FreeRTOS V'], 'FreeRTOS V%s' % self.version) + self.updateSubmodulePointers() + # When baselining off a non-HEAD commit, master is left unchanged by tagging a detached HEAD, + # applying the autocommits, tagging, and pushing the new tag data to remote. + # However in the detached HEAD state we don't have a branch to push to, so we skip + if self.commit == 'HEAD': + self.pushLocalCommits() + + self.pushTag() + self.createReleaseZip() + self.createGitRelease() + info('Core release done.') def configure_argparser(): parser = ArgumentParser(description='FreeRTOS Release tool') + + parser.add_argument('git_org', + type=str, + metavar='GITHUB_ORG', + help='Git organization owner for FreeRTOS and FreeRTOS-Kernel. (i.e. "/FreeRTOS.git")') + parser.add_argument('--new-core-version', default=None, required=False, - help='FreeRTOS-Kernel Version to replace old version. (Ex. "FreeRTOS Kernel V10.4.1")') + help='FreeRTOS Standard Distribution Version to replace old version. (Ex. "FreeRTOS V202012.00")') + + parser.add_argument('--core-commit', + default='HEAD', + required=False, + metavar='GITHUB_SHA', + help='Github SHA to baseline autorelease') + + parser.add_argument('--rollback-core-version', + default=None, + required=False, + help='Reset "master" to state prior to autorelease of given core version') + + parser.add_argument('--core-repo-path', + type=str, + default=None, + required=False, + help='Instead of downloading from git, use existing local repos for autocommits') parser.add_argument('--new-kernel-version', default=None, required=False, - help='FreeRTOS-Kernel Version to replace old version. (Ex. "FreeRTOS Kernel V10.4.1")') + help='Reset "master" to just before the autorelease for the specified kernel version")') - parser.add_argument('--git-org', - default='FreeRTOS', + parser.add_argument('--kernel-commit', + default='HEAD', required=False, - help='Git organization owner for FreeRTOS and FreeRTOS-Kernel. (i.e. "/FreeRTOS.git")') + metavar='GITHUB_SHA', + help='Github SHA to baseline autorelease') + + parser.add_argument('--rollback-kernel-version', + default=None, + required=False, + help='Reset "master" to state prior to autorelease of the given kernel version') + + parser.add_argument('--kernel-repo-path', + type=str, + default=None, + required=False, + help='Instead of downloading from git, use existing local repos for autocommits') parser.add_argument('--use-git-ssh', default=False, @@ -286,27 +494,46 @@ def main(): assert 'GITHUB_TOKEN' in os.environ, 'Set env{GITHUB_TOKEN} to an authorized git PAT' mGit = Github(os.environ.get('GITHUB_TOKEN')) - # Create release or test + # Unit tests if args.unit_test: return + # Create Releases if args.new_kernel_version: - rel_kernel = KernelRelease(mGit, args.new_kernel_version, None, git_ssh=args.use_git_ssh, git_org=args.git_org) - rel_kernel.updateFileHeaderVersions() - rel_kernel.updateVersionMacros() - rel_kernel.pushAutoCommits() - rel_kernel.createGitRelease() + info('Starting kernel release...') + logIndentPush() + rel_kernel = KernelRelease(mGit, args.new_kernel_version, args.kernel_commit, git_ssh=args.use_git_ssh, + git_org=args.git_org, repo_path=args.kernel_repo_path) + rel_kernel.autoRelease() + logIndentPop() if args.new_core_version: - rel_freertos = FreertosRelease(mGit, args.new_core_version, None, git_ssh=args.use_git_ssh, git_org=args.git_org) - rel_freertos.updateFileHeaderVersions() - rel_freertos.updateSubmodulePointers() - rel_freertos.pushAutoCommits() - rel_freertos.createReleaseZip() - rel_freertos.createGitRelease() + info('Starting core release...') + logIndentPush() + rel_freertos = FreertosRelease(mGit, args.new_core_version, args.core_commit, git_ssh=args.use_git_ssh, + git_org=args.git_org, repo_path=args.core_repo_path) + rel_freertos.autoRelease() + logIndentPop() + + # Undo autoreleases + if args.rollback_kernel_version: + info('Starting kernel rollback...') + rel_kernel = KernelRelease(mGit, args.rollback_kernel_version, args.kernel_commit, git_ssh=args.use_git_ssh, + git_org=args.git_org, repo_path=args.kernel_repo_path) + logIndentPush() + rel_kernel.restorePriorToRelease() + logIndentPop() + + if args.rollback_core_version: + info('Starting core rollback...') + logIndentPush() + rel_freertos = FreertosRelease(mGit, args.rollback_core_version, args.core_commit, git_ssh=args.use_git_ssh, + git_org=args.git_org, repo_path=args.core_repo_path) + rel_freertos.restorePriorToRelease() + logIndentPop() info('Review script output for any unexpected behaviour.') - info('Release done.') + info('Done.') if __name__ == '__main__': main() diff --git a/.github/scripts/versioning.py b/.github/scripts/versioning.py index 69c17a94fb..b4607cc2eb 100755 --- a/.github/scripts/versioning.py +++ b/.github/scripts/versioning.py @@ -69,7 +69,7 @@ def ask_yes_no_question(question): return answer -def list_files_in_a_component(component, afr_path, exclude_dirs=[], ext_filter=['.c', '.h']): +def list_files_in_a_component(component, afr_path, exclude_dirs=[], ext_filter=['.c', '.h'], exclude_hidden=True): ''' Returns a list of all the files in a component. ''' @@ -77,19 +77,17 @@ def list_files_in_a_component(component, afr_path, exclude_dirs=[], ext_filter=[ search_path = os.path.join(afr_path, component) for root, dirs, files in os.walk(search_path, topdown=True): - dirs[:] = [d for d in dirs if d not in exclude_dirs] - - # Do not include hidden files and folders. - dirs[:] = [d for d in dirs if not d[0] == '.'] - files = [f for f in files if not f[0] == '.'] + # Current root is an excluded dir so skip + if root in exclude_dirs: + continue for f in files: - if ext_filter != None: - ext = '.' + f.split('.')[-1] - if ext in ext_filter: + if exclude_hidden and f[0] == '.': + continue + + if (ext_filter is None + or ext_filter is not None and os.path.splitext(f)[-1] in ext_filter): list_of_files.append(os.path.join(os.path.relpath(root, afr_path), f)) - else: - list_of_files.append(os.path.join(os.path.relpath(root, afr_path), f)) return list_of_files @@ -104,6 +102,8 @@ def extract_version_number_from_file(file_path): # Is it a kernel file? if match is None: match = re.search('\s*\*\s*(FreeRTOS Kernel.*V(.*))', content, re.MULTILINE) + if match is None: + match = re.search('\s*\*\s*(FreeRTOS V(.*\..*))', content, re.MULTILINE) # Is it s FreeRTOS+TCP file? if match is None: match = re.search('\s*\*\s*(FreeRTOS\+TCP.*V(.*))', content, re.MULTILINE) @@ -194,7 +194,6 @@ def process_components(root_dir, components, exclude_dirs=[]): update_version_number_in_a_component(c, root_dir, exclude_dirs=exclude_dirs) def update_freertos_version_macros(path_macrofile, major, minor, build): - print('\nUpdating preprocessor version macros...') with open(path_macrofile, encoding='utf-8', errors='ignore', newline='') as macro_file: macro_file_content = macro_file.read() match_version = re.search(r'(^.*#define *tskKERNEL_VERSION_NUMBER *(".*")$)', macro_file_content, re.MULTILINE) @@ -222,12 +221,10 @@ def update_freertos_version_macros(path_macrofile, major, minor, build): with open(path_macrofile, 'w', newline='') as macro_file: macro_file.write(macro_file_content) - print('Done. Replaced "%s" --> "V%s.%s.%s".' % (old_version_number, major, minor, build)) - -def update_version_number_in_freertos_component(component, root_dir, old_version_prefix_list, new_version, verbose=False): +def update_version_number_in_freertos_component(component, root_dir, old_version_prefix_list, new_version, + verbose=False, exclude_hidden=True): assert isinstance(old_version_prefix_list, list), 'Expected a list for arg(old_version_prefix_list)' - print('Updating "%s"...' % component) - component_files = list_files_in_a_component(component, root_dir, ext_filter=None) + component_files = list_files_in_a_component(component, root_dir, ext_filter=None, exclude_hidden=exclude_hidden) version_numbers = defaultdict(list) n_updated = 0 @@ -257,7 +254,6 @@ def update_version_number_in_freertos_component(component, root_dir, old_version update_version_number_in_files(files_using_old_version, old_version_string, new_version_string) n_updated += len(files_using_old_version) - print('Updated "%d" files.' % n_updated) return n_updated def process_freertos_components(root_dir, components, old_version, new_version, verbose=False): diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml new file mode 100644 index 0000000000..f63611b17c --- /dev/null +++ b/.github/workflows/auto-release.yml @@ -0,0 +1,51 @@ +name: Auto-Release + +on: + workflow_dispatch: + inputs: + commit_id: + description: 'Commit ID' + required: true + default: 'HEAD' + version_number: + description: 'Version Number (Ex. 202000.00)' + required: true + +jobs: + auto-release: + name: Auto Release + runs-on: ubuntu-latest + steps: + - name: Tool Setup + uses: actions/setup-python@v2 + with: + python-version: 3.8.5 + architecture: x64 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Source the release tools from FreeRTOS/FreeRTOS + - name: Checkout FreeRTOS Release Tools + uses: actions/checkout@v2 + with: + path: tools + + # Simpler git auth if we use checkout action and forward the repo to release script + - name: Checkout FreeRTOS + uses: actions/checkout@v2 + with: + path: local_core + fetch-depth: 0 + + - name: Release + run: | + # Configure repo for push + git config --global user.name ${{ github.actor }} + git config --global user.email ${{ github.actor }}@users.noreply.github.com + + # Run the release script + pip install -r ./tools/.github/scripts/release-requirements.txt + ./tools/.github/scripts/release.py FreeRTOS --core-repo-path=local_core --core-commit=${{ github.event.inputs.commit_id }} --new-core-version=${{ github.event.inputs.version_number }} + exit $? + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/header-checks.yml b/.github/workflows/core-checks.yml similarity index 81% rename from .github/workflows/header-checks.yml rename to .github/workflows/core-checks.yml index 5b1f6371e2..1460c8d29b 100644 --- a/.github/workflows/header-checks.yml +++ b/.github/workflows/core-checks.yml @@ -1,10 +1,10 @@ -name: FreeRTOS-Header-Checker +name: Core-Checker on: [pull_request] jobs: - header-checker: - name: File Header Checks + core-checker: + name: FreeRTOS Core Checks runs-on: ubuntu-latest steps: # Install python 3 @@ -40,6 +40,7 @@ jobs: - name: Check File Headers run: | cd inspect - ../tools/.github/scripts/check-header.py --json ${HOME}/files_modified.json ${HOME}/files_added.json ${HOME}/files_renamed.json + export PYTHONPATH=tools/.github/scripts:${PYTHONPATH} + .github/scripts/core_checker.py --json ${HOME}/files_modified.json ${HOME}/files_added.json ${HOME}/files_renamed.json exit $? diff --git a/.github/workflows/release-packager.yml b/.github/workflows/release-packager.yml deleted file mode 100644 index 2cc7451a6d..0000000000 --- a/.github/workflows/release-packager.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: FreeRTOS-Release-Packager - -on: - workflow_dispatch: - inputs: - commit_id: - description: 'Commit ID' - required: true - version_number: - description: 'Version Number (Ex. 10.4.1)' - required: true - default: '10.4.1' - -jobs: - release-packager: - name: Release Packager - runs-on: ubuntu-latest - steps: - # Need a separate copy to fetch packing tools, as source FreeRTOS dir will be pruned and operated on - - name: Checkout FreeRTOS Tools - uses: actions/checkout@v2 - with: - ref: master - path: tools - - # Setup packing tools - - name: Tool Setup - uses: actions/setup-python@v2 - with: - python-version: 3.8.5 - architecture: x64 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # Packaging - - name: Packaging - run: python tools/.github/scripts/freertos_zipper.py --freertos-commit ${{ github.event.inputs.commit_id }} --zip-version ${{ github.event.inputs.version_number }} - - # Create release endpoint - - name: Create Release - id: create_release - uses: actions/create-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: V${{ github.event.inputs.version_number }} - release_name: FreeRTOS Release V${{ github.event.inputs.version_number }} - draft: false - prerelease: false - commitish: ${{ github.event.inputs.commit_id }} - - # Upload release assets the recently created endpoint - - name: Upload Release - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ./FreeRTOSv${{ github.event.inputs.version_number }}.zip - asset_name: FreeRTOSv${{ github.event.inputs.version_number }}.zip - asset_content_type: application/zip -