GitAction - Release Packager (#342)

* Rev0 - Release packaging action
* freertos_zipper += commit id param
+force checkout+clean required for older commits
* require commit id

.github/scripts/freertos_zipper.py

@@ -0,0 +1,280 @@
+#!/usr/bin/env python3
+
+import os, sys
+from argparse import ArgumentParser
+import shutil
+from zipfile import ZipFile
+import subprocess
+
+FREERTOS_GIT_LINK = 'https://github.com/FreeRTOS/FreeRTOS.git'
+LABS_GIT_LINK = 'https://github.com/FreeRTOS/FreeRTOS-Labs.git'
+
+DIR_INTERMEDIATE_FILES = os.path.join(os.path.basename(__file__).replace('.py', '-tmp-output'))
+DIR_INPUT_TREES = os.path.join(DIR_INTERMEDIATE_FILES, 'baseline')
+DIR_OUTPUT_TREES = os.path.join(DIR_INTERMEDIATE_FILES, 'git-head-master')
+
+RELATIVE_FILE_EXCLUDES = [
+    os.path.join('.git'),
+    os.path.join('.github'),
+    os.path.join('.gitignore'),
+    os.path.join('.gitmodules'),
+    os.path.join('CONTRIBUTING.md'),
+    os.path.join('LICENSE.md'),
+    os.path.join('README.md'),
+
+    os.path.join('FreeRTOS', 'Source', '.git'),
+    os.path.join('FreeRTOS', 'Source', '.github'),
+    os.path.join('FreeRTOS', 'Source', 'CONTRIBUTING.md'),
+    os.path.join('FreeRTOS', 'Source', 'GitHub-FreeRTOS-Kernel-Home.url'),
+    os.path.join('FreeRTOS', 'Source', 'History.txt'),
+    os.path.join('FreeRTOS', 'Source', 'LICENSE.md'),
+    os.path.join('FreeRTOS', 'Source', 'Quick_Start_Guide.url'),
+    os.path.join('FreeRTOS', 'Source', 'README.md'),
+    os.path.join('FreeRTOS', 'Source', 'SECURITY.md'),
+]
+
+LABS_RELATIVE_EXCLUDE_FILES = [
+    os.path.join('.git')
+]
+
+'''
+- Take inputs 
+    - version will be specified
+        - This will be used to name directory
+    - baseline zip. From last release
+        - Used to compare contents of last release
+- setup stage
+    - Create 'zipper-output' directory
+        - This will house the zip
+        - and the directory used to zip. 'out/unzipped-package'
+    - Unzip the input zip into 'out/baseline'
+    - Git clone recursive into latest FreeRTOS master from Git into 'out/head-master
+
+- process stage
+    - remove all RELATIVE_FILE_EXCLUDES from 'out/unzipped-package'
+    - perform a filetree diff between 'out/unzipped-package' and 'out/head-master'
+        - Save to a file, to be confirmed by user
+        - Present and query user to authorize the diff
+    - zip contents of 'out/unzipped-package' --> 'out/FreeRTOSVXX.YY.ZZ.zip
+        - Use 7z compression, with compression set to max
+    - calculate zip file size diff, present to user
+    - Done
+'''
+
+# -------------------------------------------------------------------------------------------------
+# Helpers
+# -------------------------------------------------------------------------------------------------
+def info(msg):
+    print('[INFO]: %s' % str(msg))
+
+def authorize_filetree_diff():
+    '''
+    Presents the filetree diff between baseline zip and resulting zip contents.
+    Then queries a 'y/n' response from user, to verify file diff.
+    This does not consider files that were pruned from result filetree and is to instead show
+
+    Return boolean True if user authorizes the diff, else False
+    '''
+    info('TODO')
+
+def get_file_bytesize_diff(path_newfile, path_basefile):
+    return os.path.getsize(path_newfile) - os.path.getsize(path_basefile)
+
+# -------------------------------------------------------------------------------------------------
+# Core
+# -------------------------------------------------------------------------------------------------
+def cleanup_intermediate_files(scratch_dir):
+    '''
+    Undo and cleanup actions done by 'setup_intermediate_files()'
+    '''
+    if os.path.exists(scratch_dir):
+        shutil.rmtree(scratch_dir)
+
+def unzip_baseline_zip(path_inzip, path_outdir):
+    '''
+    Unzips baseline zip into intermediate files directory. The baseline zip is used to compare against
+    resulting output zip and its contents, to produce filetree diffs, size diffs, or other diagnostics
+    '''
+    with ZipFile(path_inzip, 'r') as inzip:
+        inzip.extractall(path_outdir)
+
+    return os.path.join(path_outdir, str(os.path.basename(path_inzip)).replace('.zip', ''))
+
+def download_git_tree(git_link, root_dir, dir_name, ref='master', commit_id='HEAD'):
+    '''
+    Download HEAD from Git Master. Place into working files dir
+    '''
+    rc = subprocess.run(['git', '-C', root_dir, 'clone', '-b', ref, git_link, dir_name]).returncode
+    rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'checkout', '-f', commit_id]).returncode    
+    rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'clean', '-fd']).returncode    
+    rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'submodule', 'update', '--init', '--recursive']).returncode    
+
+    return os.path.join(root_dir, dir_name) if rc == 0 else None
+
+def setup_intermediate_files(scratch_dir, intree_dir, outtree_dir):
+    cleanup_intermediate_files(scratch_dir)
+    os.mkdir(scratch_dir)
+    os.mkdir(intree_dir)
+    os.mkdir(outtree_dir)
+
+def create_file_trees(intree_dir, baseline_zip, outtree_dir, git_link, outtree_name, git_ref='master', commit_id='HEAD'):
+    path_in_tree = None
+    path_out_tree = None
+
+    # Input baseline file tree
+    if baseline_zip != None:
+        print("Unzipping baseline: '%s'..." % baseline_zip)
+        path_in_tree = unzip_baseline_zip(baseline_zip, intree_dir)
+        print('Done.')
+
+    # Output file tree to be pruned and packaged
+    path_out_tree = download_git_tree(git_link, outtree_dir, outtree_name, commit_id=commit_id)
+
+    return (path_in_tree, path_out_tree)
+
+def prune_result_tree(path_root, exclude_files=[], dry_run=False):
+    '''
+    Remove all files specifed in 'exclude_files' from intermediate result file tree.
+    Paths in 'exclude_files' are taken relative to path_root
+    '''
+    files_removed = []
+    for f in exclude_files:
+        path_full = os.path.join(path_root, f)
+        if os.path.exists(path_full):
+            if os.path.isfile(path_full):
+                if not dry_run:
+                    os.remove(path_full)
+                files_removed.append(path_full)
+            else:
+                if not dry_run:
+                    shutil.rmtree(path_full)
+                files_removed.append(path_full)
+
+    return files_removed
+
+def zip_result_tree(path_tree, path_outzip):
+    '''
+    Zip file tree rooted at 'path_root', using same compression as 7z at max compression,
+    to zip at 'path_outzip'
+    '''
+    subprocess.run(['7z', 'a', '-tzip', '-mx=9', path_outzip, os.path.join('.', path_tree)])
+
+def show_package_diagnostics(path_newzip, path_basezip):
+    '''
+    Show various diagnostics about resulting package zip including Byte-size diff from baseline
+    and a path to
+    '''
+    if path_basezip:
+        size_diff_KB = get_file_bytesize_diff(path_newzip, path_basezip) / 1024
+        print('\nPackage growth from baseline:\n    size(%s) - size(%s) = %s%.2d KB' %
+              (path_newzip,
+               path_basezip,
+               '+' if size_diff_KB >= 0 else '', size_diff_KB))
+
+def create_package(path_outtree, package_name, exclude_files=[]):
+    print("Packaging '%s'..." % package_name)
+    pruned_files = prune_result_tree(path_outtree, exclude_files)
+    print('Files removed:\n    %s' % '\n    '.join(pruned_files))
+
+    path_outzip = '%s.zip' % package_name
+    zip_result_tree(path_outtree, path_outzip)
+    print('Done.')
+
+    return path_outzip
+# -------------------------------------------------------------------------------------------------
+# CLI
+# -------------------------------------------------------------------------------------------------
+def configure_argparser():
+    parser = ArgumentParser(description = 'Zip packaging tool for FreeRTOS release.')
+
+    parser.add_argument('--core-input-zip',
+                        metavar = 'CORE-BASELINE.ZIP',
+                        default = None,
+                        help = 'FreeRTOS baseline zip to compare against new core zip')
+
+    parser.add_argument('--labs-input-zip',
+                        metavar = 'LABS-BASELINE.ZIP',
+                        default = None,
+                        help = 'FreeRTOS-Labs baseline zip to compare agains new labs zip')
+
+    parser.add_argument('--zip-version',
+                        metavar = 'PACKAGE_VERSION_NUMBER',
+                        type = str,
+                        default = None,
+                        help = 'Version number to be suffixed to FreeRTOS and FreeRTOS-Labs zips')
+
+    parser.add_argument('--freertos-commit',
+                        metavar = 'FREERTOS_COMMIT_ID',
+                        type = str,
+                        default = 'HEAD',
+                        help = 'Commit ID of FreeRTOS repo to package')
+    
+    return parser
+
+def sanitize_cmd_args(args):
+    # Check FreeRTOS Core options
+    if not args.core_input_zip:
+        info('No FreeRTOS baseline zip provided. Zip-comparison diagnostics will not be provided...')
+        args.core_input_zip = None
+    elif not os.path.exists(args.core_input_zip):
+        error('Input zip does not exist: %s' % args.core_input_zip)
+        exit(1)
+
+    # Check FreeRTOS Labs options
+    if not args.labs_input_zip:
+        info('No FreeRTOS-Labs baseline zip provided. Zip-comparison diagnostics will not be provided...')
+        args.labs_input_zip = None
+    elif not os.path.exists(args.labs_input_zip):
+        error('Input zip does not exist: %s' % args.input_zip)
+        exit(1)
+
+    # Check version options
+    if args.zip_version == None:
+        info('No version string provide. Will use "XX.YY.ZZ" as version suffix...')
+        args.zip_version = 'XX.YY.ZZ'
+
+
+def main():
+    # CLI
+    cmd = configure_argparser()
+
+    # Setup
+    args = cmd.parse_args()
+    sanitize_cmd_args(args)
+    setup_intermediate_files(DIR_INTERMEDIATE_FILES, DIR_INPUT_TREES, DIR_OUTPUT_TREES)
+
+    # Create FreeRTOS and FreeRTOS-Labs packages
+    core_package_name = 'FreeRTOSv%s' % args.zip_version
+    (path_core_in_tree, path_core_out_tree) = create_file_trees(DIR_INPUT_TREES,
+                                                                args.core_input_zip,
+                                                                DIR_OUTPUT_TREES,
+                                                                FREERTOS_GIT_LINK,
+                                                                core_package_name,
+                                                                commit_id=args.freertos_commit)
+    
+    if path_core_out_tree == None:
+        print('Failed to prepare repo for zipping')
+        exit(1);
+    
+    core_outzip = create_package(path_core_out_tree, core_package_name, RELATIVE_FILE_EXCLUDES)
+    
+    # Create FreeRTOS-Labs package
+    labs_package_name = 'FreeRTOS-Labs'
+    (path_labs_in_tree, path_labs_out_tree) = create_file_trees(DIR_INPUT_TREES,
+                                                                args.labs_input_zip,
+                                                                DIR_OUTPUT_TREES,
+                                                                LABS_GIT_LINK,
+                                                                labs_package_name)
+    if path_labs_out_tree == None:
+        print('Failed to prepare repo for zipping')
+        exit(1);
+    
+    labs_outzip = create_package(path_labs_out_tree, labs_package_name, LABS_RELATIVE_EXCLUDE_FILES)
+
+    # Package summaries
+    show_package_diagnostics(core_outzip, args.core_input_zip)
+    show_package_diagnostics(labs_outzip, args.labs_input_zip)
+
+if __name__ == '__main__':
+    main()
+

.github/scripts/version_number_update.py

@@ -0,0 +1,246 @@
+import os
+import re
+import argparse
+from collections import defaultdict
+
+
+_AFR_COMPONENTS = [
+    'demos',
+    'freertos_kernel',
+    os.path.join('libraries','abstractions','ble_hal'),
+    os.path.join('libraries','abstractions','common_io'),
+    os.path.join('libraries','abstractions','pkcs11'),
+    os.path.join('libraries','abstractions','platform'),
+    os.path.join('libraries','abstractions','posix'),
+    os.path.join('libraries','abstractions','secure_sockets'),
+    os.path.join('libraries','abstractions','wifi'),
+    os.path.join('libraries','c_sdk','aws','defender'),
+    os.path.join('libraries','c_sdk','aws','shadow'),
+    os.path.join('libraries','c_sdk','standard','ble'),
+    os.path.join('libraries','c_sdk','standard','common'),
+    os.path.join('libraries','c_sdk','standard','https'),
+    os.path.join('libraries','c_sdk','standard','mqtt'),
+    os.path.join('libraries','c_sdk','standard','serializer'),
+    os.path.join('libraries','freertos_plus','aws','greengrass'),
+    os.path.join('libraries','freertos_plus','aws','ota'),
+    os.path.join('libraries','freertos_plus','standard','crypto'),
+    os.path.join('libraries','freertos_plus','standard','freertos_plus_posix'),
+    os.path.join('libraries','freertos_plus','standard','freertos_plus_tcp'),
+    os.path.join('libraries','freertos_plus','standard','pkcs11'),
+    os.path.join('libraries','freertos_plus','standard','tls'),
+    os.path.join('libraries','freertos_plus','standard','utils'),
+    'tests'
+]
+
+
+def ask_question(question):
+    answer = input('{}: '.format(question))
+    return answer.strip()
+
+
+def ask_multiple_choice_question(question, choices):
+    while True:
+        print('{}?'.format(question))
+        for i in range(len(choices)):
+            print('{}. {}'.format(i, choices[i]))
+        try:
+            user_choice = int(ask_question('Enter Choice'))
+        except ValueError:
+            print('Incorrect choice. Please choose a number between 0 and {}'.format(len(choices) - 1))
+            continue
+        if user_choice in range(len(choices)):
+            break
+        else:
+            print('Incorrect choice. Please choose a number between 0 and {}'.format(len(choices) - 1))
+    return user_choice
+
+
+def ask_yes_no_question(question):
+    while True:
+        answer = ask_question('{} (Y/N)'.format(question))
+        if answer.lower() == 'y':
+            answer = 'yes'
+            break
+        elif answer.lower() == 'n':
+            answer = 'no'
+            break
+        else:
+            print('Incorrect response. Please answer Y/N.')
+    return answer
+
+
+def print_file_list(file_list):
+    version_line_list = []
+
+    for file in file_list:
+        version_number = extract_version_number_from_file(file)
+        version_line_list.append(version_number[0] if version_number[0] is not None else 'Could not detect version')
+
+    max_filepath_length = len(max(file_list, key=len))
+    max_version_line_length = len(max(version_line_list, key=len))
+
+    print('-' * (max_filepath_length + max_version_line_length + 7))
+    print('| {file:<{max_filepath_length}} | {version:<{max_version_line_length}} |'.format(file='File',
+                                                                                            max_filepath_length=max_filepath_length,
+                                                                                            version='Version Line',
+                                                                                            max_version_line_length=max_version_line_length))
+    print('-' * (max_filepath_length + max_version_line_length + 7))
+    for i in range(len(file_list)):
+        print('| {file:<{max_filepath_length}} | {version:<{max_version_line_length}} |'.format(file=file_list[i],
+                                                                                                max_filepath_length=max_filepath_length,
+                                                                                                version=version_line_list[i],
+                                                                                                max_version_line_length=max_version_line_length))
+
+    print('-' * (max_filepath_length + max_version_line_length + 7))
+    print('\n')
+
+
+def list_files_in_a_component(component, afr_path):
+    '''
+    Returns a list of all the files in a component.
+    '''
+    list_of_files = []
+    search_path = os.path.join(afr_path, component)
+
+    for root, dirs, files in os.walk(search_path, topdown=True):
+        # Do not search 'portable' and 'third_party' folders.
+        dirs[:] = [d for d in dirs if d not in ['portable', 'third_party']]
+
+        # Do not include hidden files and folders.
+        dirs[:] = [d for d in dirs if not d[0] == '.']
+        files = [f for f in files if not f[0] == '.']
+
+        for f in files:
+            if f.endswith('.c') or f.endswith('.h'):
+                list_of_files.append(os.path.join(os.path.relpath(root, afr_path), f))
+
+    return list_of_files
+
+
+def extract_version_number_from_file(file_path):
+    '''
+    Extracts version number from the License header in a file.
+    '''
+    with open(file_path) as f:
+        content = f.read()
+        match = re.search('\s*\*\s*(FreeRTOS.*V(.*))', content, re.MULTILINE)
+        # Is it a kernel file?
+        if match is None:
+            match = re.search('\s*\*\s*(FreeRTOS Kernel.*V(.*))', content, re.MULTILINE)
+        # Is it s FreeRTOS+TCP file?
+        if match is None:
+            match = re.search('\s*\*\s*(FreeRTOS\+TCP.*V(.*))', content, re.MULTILINE)
+    return (match.group(1), match.group(2)) if match is not None else (None, None)
+
+
+def update_version_number_in_files(file_paths, old_version_line, new_version_line):
+    '''
+    Replaces old_version_line with new_version_line in all the files specified
+    by file_paths.
+    '''
+    for file_path in file_paths:
+        with open(file_path) as f:
+            content = f.read()
+            content = content.replace(old_version_line, new_version_line)
+        with open(file_path, 'w') as f:
+            f.write(content)
+
+
+def update_version_number_in_a_component(component, afr_path):
+    '''
+    Updates version numbers in all the files of an AFR component based on user
+    choices.
+    '''
+    # Get all the files in the component.
+    files_in_component = list_files_in_a_component(component, afr_path)
+
+    version_numbers = defaultdict(list)
+
+    # Extract version numbers from all the files.
+    for f in files_in_component:
+        file_path = os.path.join(afr_path, f)
+        version_number = extract_version_number_from_file(file_path)
+        version_numbers[version_number].append(file_path)
+
+    for key in version_numbers.keys():
+        old_version_line = key[0]
+        old_version_number = key[1]
+        files_to_update = version_numbers[key]
+
+        if old_version_line is None:
+            print('\nFailed to detect the version number in the following files:')
+            while True:
+                print_file_list(files_to_update)
+                print('Please update the above files manually!')
+                confirm = ask_yes_no_question('Done updating')
+                if confirm == 'yes':
+                    print_file_list(files_to_update)
+                    looks_good = ask_yes_no_question('Does it look good')
+                    if looks_good == 'yes':
+                        break
+        else:
+            print('\n{} files have the following version: {}\n'.format(len(files_to_update), old_version_line))
+
+            options = [ 'Update version number [i.e. update "{}"].'.format(old_version_number),
+                        'Update version line [i.e. update "{}"].'.format(old_version_line),
+                        'List files.',
+                        'Do not update.' ]
+
+            while True:
+                user_selected_option = ask_multiple_choice_question('What do you want to do', options)
+
+                if user_selected_option == 0:
+                    new_version_number = ask_question('Enter new version number')
+                    new_version_line = old_version_line.replace(old_version_number, new_version_number)
+                    print('Old version line: "{}". New version line: "{}".'.format(old_version_line, new_version_line))
+                    confirm = ask_yes_no_question('Does it look good')
+                    if confirm == 'yes':
+                        update_version_number_in_files(files_to_update, old_version_line, new_version_line)
+                        print('Updated version line to "{}".\n'.format(new_version_line))
+                        break
+                elif user_selected_option == 1:
+                    new_version_line = ask_question('Enter new version line')
+                    print('Old version line: "{}". New version line: "{}".'.format(old_version_line, new_version_line))
+                    confirm = ask_yes_no_question('Does it look good')
+                    if confirm == 'yes':
+                        update_version_number_in_files(files_to_update, old_version_line, new_version_line)
+                        print('Updated version line to "{}".\n'.format(new_version_line))
+                        break
+                elif user_selected_option == 2:
+                    print_file_list(files_to_update)
+                else:
+                    print('Skipping update of {}.\n'.format(old_version_line))
+                    break
+
+
+def parse_arguments():
+    '''
+    Parses the command line arguments.
+    '''
+    parser = argparse.ArgumentParser(description='FreeRTOS Checksum Generator')
+    parser.add_argument('--afr', required=True, help='Location of the AFR Code.')
+    args = parser.parse_args()
+    return vars(args)
+
+
+def main():
+    '''
+    Main entry point.
+    '''
+    args = parse_arguments()
+
+    afr_path = args['afr']
+
+    print('AFR Code: {}'.format(afr_path))
+
+    for component in _AFR_COMPONENTS:
+        print('\n---------------------------------------------')
+        print('Component: {}'.format(component))
+        print('---------------------------------------------\n')
+        wanna_update_version = ask_yes_no_question('Do you want to update the component "{}"'.format(component))
+        if wanna_update_version == 'yes':
+            update_version_number_in_a_component(component, afr_path)
+
+
+if __name__ == '__main__':
+    main()

.github/workflows/release-packager.yml

@@ -0,0 +1,62 @@
+name: FreeRTOS-Release-Packager
+
+on:
+  workflow_dispatch:
+    inputs:
+      commit_id:
+        description: 'Commit ID'
+        required: true
+      version_number:
+        description: 'Version Number (Ex. 10.4.1)'
+        required: true
+        default: '10.4.1'             
+
+jobs:
+  release-packager:
+    name: Release Packager
+    runs-on: ubuntu-latest
+    steps:
+      # Need a separate copy to fetch packing tools, as source FreeRTOS dir will be pruned and operated on
+      - name: Checkout FreeRTOS Tools
+        uses: actions/checkout@v2
+        with:
+          ref:  master
+          path: tools          
+          
+      # Setup packing tools
+      - name: Tool Setup
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8.5
+          architecture:   x64  
+        env:  
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 
+      
+      # Packaging     
+      - name: Packaging 
+        run: python tools/.github/scripts/freertos_zipper.py --freertos-commit ${{ github.event.inputs.commit_id }} --zip-version ${{ github.event.inputs.version_number }} 
+                  
+      # Create release endpoint
+      - name: Create Release
+        id:   create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+            tag_name:     V${{ github.event.inputs.version_number }} 
+            release_name: FreeRTOS Release V${{ github.event.inputs.version_number }} 
+            draft:        false
+            prerelease:   false
+            commitish:    ${{ github.event.inputs.commit_id }}
+                      
+      # Upload release assets the recently created endpoint
+      - name: Upload Release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+            upload_url: ${{ steps.create_release.outputs.upload_url }}
+            asset_path: ./FreeRTOSv${{ github.event.inputs.version_number }}.zip
+            asset_name: FreeRTOSv${{ github.event.inputs.version_number }}.zip
+            asset_content_type: application/zip
+