From d47a28aff7c869f3659fb90a8ccf0cbe3c5e4ac0 Mon Sep 17 00:00:00 2001 From: David Chalco <59750547+dachalco@users.noreply.github.com> Date: Fri, 16 Oct 2020 10:19:02 -0700 Subject: [PATCH] GitAction - Release Packager (#342) * Rev0 - Release packaging action * freertos_zipper += commit id param +force checkout+clean required for older commits * require commit id --- .github/scripts/freertos_zipper.py | 280 +++++++++++++++++++++++ .github/scripts/version_number_update.py | 246 ++++++++++++++++++++ .github/workflows/release-packager.yml | 62 +++++ 3 files changed, 588 insertions(+) create mode 100755 .github/scripts/freertos_zipper.py create mode 100644 .github/scripts/version_number_update.py create mode 100644 .github/workflows/release-packager.yml diff --git a/.github/scripts/freertos_zipper.py b/.github/scripts/freertos_zipper.py new file mode 100755 index 0000000000..a46271ecf3 --- /dev/null +++ b/.github/scripts/freertos_zipper.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 + +import os, sys +from argparse import ArgumentParser +import shutil +from zipfile import ZipFile +import subprocess + +FREERTOS_GIT_LINK = 'https://github.com/FreeRTOS/FreeRTOS.git' +LABS_GIT_LINK = 'https://github.com/FreeRTOS/FreeRTOS-Labs.git' + +DIR_INTERMEDIATE_FILES = os.path.join(os.path.basename(__file__).replace('.py', '-tmp-output')) +DIR_INPUT_TREES = os.path.join(DIR_INTERMEDIATE_FILES, 'baseline') +DIR_OUTPUT_TREES = os.path.join(DIR_INTERMEDIATE_FILES, 'git-head-master') + +RELATIVE_FILE_EXCLUDES = [ + os.path.join('.git'), + os.path.join('.github'), + os.path.join('.gitignore'), + os.path.join('.gitmodules'), + os.path.join('CONTRIBUTING.md'), + os.path.join('LICENSE.md'), + os.path.join('README.md'), + + os.path.join('FreeRTOS', 'Source', '.git'), + os.path.join('FreeRTOS', 'Source', '.github'), + os.path.join('FreeRTOS', 'Source', 'CONTRIBUTING.md'), + os.path.join('FreeRTOS', 'Source', 'GitHub-FreeRTOS-Kernel-Home.url'), + os.path.join('FreeRTOS', 'Source', 'History.txt'), + os.path.join('FreeRTOS', 'Source', 'LICENSE.md'), + os.path.join('FreeRTOS', 'Source', 'Quick_Start_Guide.url'), + os.path.join('FreeRTOS', 'Source', 'README.md'), + os.path.join('FreeRTOS', 'Source', 'SECURITY.md'), +] + +LABS_RELATIVE_EXCLUDE_FILES = [ + os.path.join('.git') +] + +''' +- Take inputs + - version will be specified + - This will be used to name directory + - baseline zip. From last release + - Used to compare contents of last release +- setup stage + - Create 'zipper-output' directory + - This will house the zip + - and the directory used to zip. 'out/unzipped-package' + - Unzip the input zip into 'out/baseline' + - Git clone recursive into latest FreeRTOS master from Git into 'out/head-master + +- process stage + - remove all RELATIVE_FILE_EXCLUDES from 'out/unzipped-package' + - perform a filetree diff between 'out/unzipped-package' and 'out/head-master' + - Save to a file, to be confirmed by user + - Present and query user to authorize the diff + - zip contents of 'out/unzipped-package' --> 'out/FreeRTOSVXX.YY.ZZ.zip + - Use 7z compression, with compression set to max + - calculate zip file size diff, present to user + - Done +''' + +# ------------------------------------------------------------------------------------------------- +# Helpers +# ------------------------------------------------------------------------------------------------- +def info(msg): + print('[INFO]: %s' % str(msg)) + +def authorize_filetree_diff(): + ''' + Presents the filetree diff between baseline zip and resulting zip contents. + Then queries a 'y/n' response from user, to verify file diff. + This does not consider files that were pruned from result filetree and is to instead show + + Return boolean True if user authorizes the diff, else False + ''' + info('TODO') + +def get_file_bytesize_diff(path_newfile, path_basefile): + return os.path.getsize(path_newfile) - os.path.getsize(path_basefile) + +# ------------------------------------------------------------------------------------------------- +# Core +# ------------------------------------------------------------------------------------------------- +def cleanup_intermediate_files(scratch_dir): + ''' + Undo and cleanup actions done by 'setup_intermediate_files()' + ''' + if os.path.exists(scratch_dir): + shutil.rmtree(scratch_dir) + +def unzip_baseline_zip(path_inzip, path_outdir): + ''' + Unzips baseline zip into intermediate files directory. The baseline zip is used to compare against + resulting output zip and its contents, to produce filetree diffs, size diffs, or other diagnostics + ''' + with ZipFile(path_inzip, 'r') as inzip: + inzip.extractall(path_outdir) + + return os.path.join(path_outdir, str(os.path.basename(path_inzip)).replace('.zip', '')) + +def download_git_tree(git_link, root_dir, dir_name, ref='master', commit_id='HEAD'): + ''' + Download HEAD from Git Master. Place into working files dir + ''' + rc = subprocess.run(['git', '-C', root_dir, 'clone', '-b', ref, git_link, dir_name]).returncode + rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'checkout', '-f', commit_id]).returncode + rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'clean', '-fd']).returncode + rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'submodule', 'update', '--init', '--recursive']).returncode + + return os.path.join(root_dir, dir_name) if rc == 0 else None + +def setup_intermediate_files(scratch_dir, intree_dir, outtree_dir): + cleanup_intermediate_files(scratch_dir) + os.mkdir(scratch_dir) + os.mkdir(intree_dir) + os.mkdir(outtree_dir) + +def create_file_trees(intree_dir, baseline_zip, outtree_dir, git_link, outtree_name, git_ref='master', commit_id='HEAD'): + path_in_tree = None + path_out_tree = None + + # Input baseline file tree + if baseline_zip != None: + print("Unzipping baseline: '%s'..." % baseline_zip) + path_in_tree = unzip_baseline_zip(baseline_zip, intree_dir) + print('Done.') + + # Output file tree to be pruned and packaged + path_out_tree = download_git_tree(git_link, outtree_dir, outtree_name, commit_id=commit_id) + + return (path_in_tree, path_out_tree) + +def prune_result_tree(path_root, exclude_files=[], dry_run=False): + ''' + Remove all files specifed in 'exclude_files' from intermediate result file tree. + Paths in 'exclude_files' are taken relative to path_root + ''' + files_removed = [] + for f in exclude_files: + path_full = os.path.join(path_root, f) + if os.path.exists(path_full): + if os.path.isfile(path_full): + if not dry_run: + os.remove(path_full) + files_removed.append(path_full) + else: + if not dry_run: + shutil.rmtree(path_full) + files_removed.append(path_full) + + return files_removed + +def zip_result_tree(path_tree, path_outzip): + ''' + Zip file tree rooted at 'path_root', using same compression as 7z at max compression, + to zip at 'path_outzip' + ''' + subprocess.run(['7z', 'a', '-tzip', '-mx=9', path_outzip, os.path.join('.', path_tree)]) + +def show_package_diagnostics(path_newzip, path_basezip): + ''' + Show various diagnostics about resulting package zip including Byte-size diff from baseline + and a path to + ''' + if path_basezip: + size_diff_KB = get_file_bytesize_diff(path_newzip, path_basezip) / 1024 + print('\nPackage growth from baseline:\n size(%s) - size(%s) = %s%.2d KB' % + (path_newzip, + path_basezip, + '+' if size_diff_KB >= 0 else '', size_diff_KB)) + +def create_package(path_outtree, package_name, exclude_files=[]): + print("Packaging '%s'..." % package_name) + pruned_files = prune_result_tree(path_outtree, exclude_files) + print('Files removed:\n %s' % '\n '.join(pruned_files)) + + path_outzip = '%s.zip' % package_name + zip_result_tree(path_outtree, path_outzip) + print('Done.') + + return path_outzip +# ------------------------------------------------------------------------------------------------- +# CLI +# ------------------------------------------------------------------------------------------------- +def configure_argparser(): + parser = ArgumentParser(description = 'Zip packaging tool for FreeRTOS release.') + + parser.add_argument('--core-input-zip', + metavar = 'CORE-BASELINE.ZIP', + default = None, + help = 'FreeRTOS baseline zip to compare against new core zip') + + parser.add_argument('--labs-input-zip', + metavar = 'LABS-BASELINE.ZIP', + default = None, + help = 'FreeRTOS-Labs baseline zip to compare agains new labs zip') + + parser.add_argument('--zip-version', + metavar = 'PACKAGE_VERSION_NUMBER', + type = str, + default = None, + help = 'Version number to be suffixed to FreeRTOS and FreeRTOS-Labs zips') + + parser.add_argument('--freertos-commit', + metavar = 'FREERTOS_COMMIT_ID', + type = str, + default = 'HEAD', + help = 'Commit ID of FreeRTOS repo to package') + + return parser + +def sanitize_cmd_args(args): + # Check FreeRTOS Core options + if not args.core_input_zip: + info('No FreeRTOS baseline zip provided. Zip-comparison diagnostics will not be provided...') + args.core_input_zip = None + elif not os.path.exists(args.core_input_zip): + error('Input zip does not exist: %s' % args.core_input_zip) + exit(1) + + # Check FreeRTOS Labs options + if not args.labs_input_zip: + info('No FreeRTOS-Labs baseline zip provided. Zip-comparison diagnostics will not be provided...') + args.labs_input_zip = None + elif not os.path.exists(args.labs_input_zip): + error('Input zip does not exist: %s' % args.input_zip) + exit(1) + + # Check version options + if args.zip_version == None: + info('No version string provide. Will use "XX.YY.ZZ" as version suffix...') + args.zip_version = 'XX.YY.ZZ' + + +def main(): + # CLI + cmd = configure_argparser() + + # Setup + args = cmd.parse_args() + sanitize_cmd_args(args) + setup_intermediate_files(DIR_INTERMEDIATE_FILES, DIR_INPUT_TREES, DIR_OUTPUT_TREES) + + # Create FreeRTOS and FreeRTOS-Labs packages + core_package_name = 'FreeRTOSv%s' % args.zip_version + (path_core_in_tree, path_core_out_tree) = create_file_trees(DIR_INPUT_TREES, + args.core_input_zip, + DIR_OUTPUT_TREES, + FREERTOS_GIT_LINK, + core_package_name, + commit_id=args.freertos_commit) + + if path_core_out_tree == None: + print('Failed to prepare repo for zipping') + exit(1); + + core_outzip = create_package(path_core_out_tree, core_package_name, RELATIVE_FILE_EXCLUDES) + + # Create FreeRTOS-Labs package + labs_package_name = 'FreeRTOS-Labs' + (path_labs_in_tree, path_labs_out_tree) = create_file_trees(DIR_INPUT_TREES, + args.labs_input_zip, + DIR_OUTPUT_TREES, + LABS_GIT_LINK, + labs_package_name) + if path_labs_out_tree == None: + print('Failed to prepare repo for zipping') + exit(1); + + labs_outzip = create_package(path_labs_out_tree, labs_package_name, LABS_RELATIVE_EXCLUDE_FILES) + + # Package summaries + show_package_diagnostics(core_outzip, args.core_input_zip) + show_package_diagnostics(labs_outzip, args.labs_input_zip) + +if __name__ == '__main__': + main() + diff --git a/.github/scripts/version_number_update.py b/.github/scripts/version_number_update.py new file mode 100644 index 0000000000..5087121448 --- /dev/null +++ b/.github/scripts/version_number_update.py @@ -0,0 +1,246 @@ +import os +import re +import argparse +from collections import defaultdict + + +_AFR_COMPONENTS = [ + 'demos', + 'freertos_kernel', + os.path.join('libraries','abstractions','ble_hal'), + os.path.join('libraries','abstractions','common_io'), + os.path.join('libraries','abstractions','pkcs11'), + os.path.join('libraries','abstractions','platform'), + os.path.join('libraries','abstractions','posix'), + os.path.join('libraries','abstractions','secure_sockets'), + os.path.join('libraries','abstractions','wifi'), + os.path.join('libraries','c_sdk','aws','defender'), + os.path.join('libraries','c_sdk','aws','shadow'), + os.path.join('libraries','c_sdk','standard','ble'), + os.path.join('libraries','c_sdk','standard','common'), + os.path.join('libraries','c_sdk','standard','https'), + os.path.join('libraries','c_sdk','standard','mqtt'), + os.path.join('libraries','c_sdk','standard','serializer'), + os.path.join('libraries','freertos_plus','aws','greengrass'), + os.path.join('libraries','freertos_plus','aws','ota'), + os.path.join('libraries','freertos_plus','standard','crypto'), + os.path.join('libraries','freertos_plus','standard','freertos_plus_posix'), + os.path.join('libraries','freertos_plus','standard','freertos_plus_tcp'), + os.path.join('libraries','freertos_plus','standard','pkcs11'), + os.path.join('libraries','freertos_plus','standard','tls'), + os.path.join('libraries','freertos_plus','standard','utils'), + 'tests' +] + + +def ask_question(question): + answer = input('{}: '.format(question)) + return answer.strip() + + +def ask_multiple_choice_question(question, choices): + while True: + print('{}?'.format(question)) + for i in range(len(choices)): + print('{}. {}'.format(i, choices[i])) + try: + user_choice = int(ask_question('Enter Choice')) + except ValueError: + print('Incorrect choice. Please choose a number between 0 and {}'.format(len(choices) - 1)) + continue + if user_choice in range(len(choices)): + break + else: + print('Incorrect choice. Please choose a number between 0 and {}'.format(len(choices) - 1)) + return user_choice + + +def ask_yes_no_question(question): + while True: + answer = ask_question('{} (Y/N)'.format(question)) + if answer.lower() == 'y': + answer = 'yes' + break + elif answer.lower() == 'n': + answer = 'no' + break + else: + print('Incorrect response. Please answer Y/N.') + return answer + + +def print_file_list(file_list): + version_line_list = [] + + for file in file_list: + version_number = extract_version_number_from_file(file) + version_line_list.append(version_number[0] if version_number[0] is not None else 'Could not detect version') + + max_filepath_length = len(max(file_list, key=len)) + max_version_line_length = len(max(version_line_list, key=len)) + + print('-' * (max_filepath_length + max_version_line_length + 7)) + print('| {file:<{max_filepath_length}} | {version:<{max_version_line_length}} |'.format(file='File', + max_filepath_length=max_filepath_length, + version='Version Line', + max_version_line_length=max_version_line_length)) + print('-' * (max_filepath_length + max_version_line_length + 7)) + for i in range(len(file_list)): + print('| {file:<{max_filepath_length}} | {version:<{max_version_line_length}} |'.format(file=file_list[i], + max_filepath_length=max_filepath_length, + version=version_line_list[i], + max_version_line_length=max_version_line_length)) + + print('-' * (max_filepath_length + max_version_line_length + 7)) + print('\n') + + +def list_files_in_a_component(component, afr_path): + ''' + Returns a list of all the files in a component. + ''' + list_of_files = [] + search_path = os.path.join(afr_path, component) + + for root, dirs, files in os.walk(search_path, topdown=True): + # Do not search 'portable' and 'third_party' folders. + dirs[:] = [d for d in dirs if d not in ['portable', 'third_party']] + + # Do not include hidden files and folders. + dirs[:] = [d for d in dirs if not d[0] == '.'] + files = [f for f in files if not f[0] == '.'] + + for f in files: + if f.endswith('.c') or f.endswith('.h'): + list_of_files.append(os.path.join(os.path.relpath(root, afr_path), f)) + + return list_of_files + + +def extract_version_number_from_file(file_path): + ''' + Extracts version number from the License header in a file. + ''' + with open(file_path) as f: + content = f.read() + match = re.search('\s*\*\s*(FreeRTOS.*V(.*))', content, re.MULTILINE) + # Is it a kernel file? + if match is None: + match = re.search('\s*\*\s*(FreeRTOS Kernel.*V(.*))', content, re.MULTILINE) + # Is it s FreeRTOS+TCP file? + if match is None: + match = re.search('\s*\*\s*(FreeRTOS\+TCP.*V(.*))', content, re.MULTILINE) + return (match.group(1), match.group(2)) if match is not None else (None, None) + + +def update_version_number_in_files(file_paths, old_version_line, new_version_line): + ''' + Replaces old_version_line with new_version_line in all the files specified + by file_paths. + ''' + for file_path in file_paths: + with open(file_path) as f: + content = f.read() + content = content.replace(old_version_line, new_version_line) + with open(file_path, 'w') as f: + f.write(content) + + +def update_version_number_in_a_component(component, afr_path): + ''' + Updates version numbers in all the files of an AFR component based on user + choices. + ''' + # Get all the files in the component. + files_in_component = list_files_in_a_component(component, afr_path) + + version_numbers = defaultdict(list) + + # Extract version numbers from all the files. + for f in files_in_component: + file_path = os.path.join(afr_path, f) + version_number = extract_version_number_from_file(file_path) + version_numbers[version_number].append(file_path) + + for key in version_numbers.keys(): + old_version_line = key[0] + old_version_number = key[1] + files_to_update = version_numbers[key] + + if old_version_line is None: + print('\nFailed to detect the version number in the following files:') + while True: + print_file_list(files_to_update) + print('Please update the above files manually!') + confirm = ask_yes_no_question('Done updating') + if confirm == 'yes': + print_file_list(files_to_update) + looks_good = ask_yes_no_question('Does it look good') + if looks_good == 'yes': + break + else: + print('\n{} files have the following version: {}\n'.format(len(files_to_update), old_version_line)) + + options = [ 'Update version number [i.e. update "{}"].'.format(old_version_number), + 'Update version line [i.e. update "{}"].'.format(old_version_line), + 'List files.', + 'Do not update.' ] + + while True: + user_selected_option = ask_multiple_choice_question('What do you want to do', options) + + if user_selected_option == 0: + new_version_number = ask_question('Enter new version number') + new_version_line = old_version_line.replace(old_version_number, new_version_number) + print('Old version line: "{}". New version line: "{}".'.format(old_version_line, new_version_line)) + confirm = ask_yes_no_question('Does it look good') + if confirm == 'yes': + update_version_number_in_files(files_to_update, old_version_line, new_version_line) + print('Updated version line to "{}".\n'.format(new_version_line)) + break + elif user_selected_option == 1: + new_version_line = ask_question('Enter new version line') + print('Old version line: "{}". New version line: "{}".'.format(old_version_line, new_version_line)) + confirm = ask_yes_no_question('Does it look good') + if confirm == 'yes': + update_version_number_in_files(files_to_update, old_version_line, new_version_line) + print('Updated version line to "{}".\n'.format(new_version_line)) + break + elif user_selected_option == 2: + print_file_list(files_to_update) + else: + print('Skipping update of {}.\n'.format(old_version_line)) + break + + +def parse_arguments(): + ''' + Parses the command line arguments. + ''' + parser = argparse.ArgumentParser(description='FreeRTOS Checksum Generator') + parser.add_argument('--afr', required=True, help='Location of the AFR Code.') + args = parser.parse_args() + return vars(args) + + +def main(): + ''' + Main entry point. + ''' + args = parse_arguments() + + afr_path = args['afr'] + + print('AFR Code: {}'.format(afr_path)) + + for component in _AFR_COMPONENTS: + print('\n---------------------------------------------') + print('Component: {}'.format(component)) + print('---------------------------------------------\n') + wanna_update_version = ask_yes_no_question('Do you want to update the component "{}"'.format(component)) + if wanna_update_version == 'yes': + update_version_number_in_a_component(component, afr_path) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/release-packager.yml b/.github/workflows/release-packager.yml new file mode 100644 index 0000000000..2cc7451a6d --- /dev/null +++ b/.github/workflows/release-packager.yml @@ -0,0 +1,62 @@ +name: FreeRTOS-Release-Packager + +on: + workflow_dispatch: + inputs: + commit_id: + description: 'Commit ID' + required: true + version_number: + description: 'Version Number (Ex. 10.4.1)' + required: true + default: '10.4.1' + +jobs: + release-packager: + name: Release Packager + runs-on: ubuntu-latest + steps: + # Need a separate copy to fetch packing tools, as source FreeRTOS dir will be pruned and operated on + - name: Checkout FreeRTOS Tools + uses: actions/checkout@v2 + with: + ref: master + path: tools + + # Setup packing tools + - name: Tool Setup + uses: actions/setup-python@v2 + with: + python-version: 3.8.5 + architecture: x64 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # Packaging + - name: Packaging + run: python tools/.github/scripts/freertos_zipper.py --freertos-commit ${{ github.event.inputs.commit_id }} --zip-version ${{ github.event.inputs.version_number }} + + # Create release endpoint + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: V${{ github.event.inputs.version_number }} + release_name: FreeRTOS Release V${{ github.event.inputs.version_number }} + draft: false + prerelease: false + commitish: ${{ github.event.inputs.commit_id }} + + # Upload release assets the recently created endpoint + - name: Upload Release + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: ./FreeRTOSv${{ github.event.inputs.version_number }}.zip + asset_name: FreeRTOSv${{ github.event.inputs.version_number }}.zip + asset_content_type: application/zip +