#!/usr/bin/env python3 import os, sys from argparse import ArgumentParser import shutil from zipfile import ZipFile import subprocess FREERTOS_GIT_LINK = 'https://github.com/FreeRTOS/FreeRTOS.git' LABS_GIT_LINK = 'https://github.com/FreeRTOS/FreeRTOS-Labs.git' DIR_INTERMEDIATE_FILES = os.path.join(os.path.basename(__file__).replace('.py', '-tmp-output')) DIR_INPUT_TREES = os.path.join(DIR_INTERMEDIATE_FILES, 'baseline') DIR_OUTPUT_TREES = os.path.join(DIR_INTERMEDIATE_FILES, 'git-head-master') RELATIVE_FILE_EXCLUDES = [ os.path.join('.git'), os.path.join('.github'), os.path.join('.gitignore'), os.path.join('.gitmodules'), os.path.join('CONTRIBUTING.md'), os.path.join('LICENSE.md'), os.path.join('README.md'), os.path.join('FreeRTOS', 'Source', '.git'), os.path.join('FreeRTOS', 'Source', '.github'), os.path.join('FreeRTOS', 'Source', 'CONTRIBUTING.md'), os.path.join('FreeRTOS', 'Source', 'GitHub-FreeRTOS-Kernel-Home.url'), os.path.join('FreeRTOS', 'Source', 'History.txt'), os.path.join('FreeRTOS', 'Source', 'LICENSE.md'), os.path.join('FreeRTOS', 'Source', 'Quick_Start_Guide.url'), os.path.join('FreeRTOS', 'Source', 'README.md'), os.path.join('FreeRTOS', 'Source', 'SECURITY.md'), ] LABS_RELATIVE_EXCLUDE_FILES = [ os.path.join('.git') ] ''' - Take inputs - version will be specified - This will be used to name directory - baseline zip. From last release - Used to compare contents of last release - setup stage - Create 'zipper-output' directory - This will house the zip - and the directory used to zip. 'out/unzipped-package' - Unzip the input zip into 'out/baseline' - Git clone recursive into latest FreeRTOS master from Git into 'out/head-master - process stage - remove all RELATIVE_FILE_EXCLUDES from 'out/unzipped-package' - perform a filetree diff between 'out/unzipped-package' and 'out/head-master' - Save to a file, to be confirmed by user - Present and query user to authorize the diff - zip contents of 'out/unzipped-package' --> 'out/FreeRTOSVXX.YY.ZZ.zip - Use 7z compression, with compression set to max - calculate zip file size diff, present to user - Done ''' # ------------------------------------------------------------------------------------------------- # Helpers # ------------------------------------------------------------------------------------------------- def info(msg): print('[INFO]: %s' % str(msg)) def authorize_filetree_diff(): ''' Presents the filetree diff between baseline zip and resulting zip contents. Then queries a 'y/n' response from user, to verify file diff. This does not consider files that were pruned from result filetree and is to instead show Return boolean True if user authorizes the diff, else False ''' info('TODO') def get_file_bytesize_diff(path_newfile, path_basefile): return os.path.getsize(path_newfile) - os.path.getsize(path_basefile) # ------------------------------------------------------------------------------------------------- # Core # ------------------------------------------------------------------------------------------------- def cleanup_intermediate_files(scratch_dir): ''' Undo and cleanup actions done by 'setup_intermediate_files()' ''' if os.path.exists(scratch_dir): shutil.rmtree(scratch_dir) def unzip_baseline_zip(path_inzip, path_outdir): ''' Unzips baseline zip into intermediate files directory. The baseline zip is used to compare against resulting output zip and its contents, to produce filetree diffs, size diffs, or other diagnostics ''' with ZipFile(path_inzip, 'r') as inzip: inzip.extractall(path_outdir) return os.path.join(path_outdir, str(os.path.basename(path_inzip)).replace('.zip', '')) def download_git_tree(git_link, root_dir, dir_name, ref='master', commit_id='HEAD'): ''' Download HEAD from Git Master. Place into working files dir ''' rc = subprocess.run(['git', '-C', root_dir, 'clone', '-b', ref, git_link, dir_name]).returncode rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'checkout', '-f', commit_id]).returncode rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'clean', '-fd']).returncode rc += subprocess.run(['git', '-C', os.path.join(root_dir, dir_name), 'submodule', 'update', '--init', '--recursive']).returncode return os.path.join(root_dir, dir_name) if rc == 0 else None def setup_intermediate_files(scratch_dir, intree_dir, outtree_dir): cleanup_intermediate_files(scratch_dir) os.mkdir(scratch_dir) os.mkdir(intree_dir) os.mkdir(outtree_dir) def create_file_trees(intree_dir, baseline_zip, outtree_dir, git_link, outtree_name, git_ref='master', commit_id='HEAD'): path_in_tree = None path_out_tree = None # Input baseline file tree if baseline_zip != None: print("Unzipping baseline: '%s'..." % baseline_zip) path_in_tree = unzip_baseline_zip(baseline_zip, intree_dir) print('Done.') # Output file tree to be pruned and packaged path_out_tree = download_git_tree(git_link, outtree_dir, outtree_name, commit_id=commit_id) return (path_in_tree, path_out_tree) def prune_result_tree(path_root, exclude_files=[], dry_run=False): ''' Remove all files specifed in 'exclude_files' from intermediate result file tree. Paths in 'exclude_files' are taken relative to path_root ''' files_removed = [] for f in exclude_files: path_full = os.path.join(path_root, f) if os.path.exists(path_full): if os.path.isfile(path_full): if not dry_run: os.remove(path_full) files_removed.append(path_full) else: if not dry_run: shutil.rmtree(path_full) files_removed.append(path_full) return files_removed def zip_result_tree(path_tree, path_outzip): ''' Zip file tree rooted at 'path_root', using same compression as 7z at max compression, to zip at 'path_outzip' ''' subprocess.run(['7z', 'a', '-tzip', '-mx=9', path_outzip, os.path.join('.', path_tree)]) def show_package_diagnostics(path_newzip, path_basezip): ''' Show various diagnostics about resulting package zip including Byte-size diff from baseline and a path to ''' if path_basezip: size_diff_KB = get_file_bytesize_diff(path_newzip, path_basezip) / 1024 print('\nPackage growth from baseline:\n size(%s) - size(%s) = %s%.2d KB' % (path_newzip, path_basezip, '+' if size_diff_KB >= 0 else '', size_diff_KB)) def create_package(path_outtree, package_name, exclude_files=[]): print("Packaging '%s'..." % package_name) pruned_files = prune_result_tree(path_outtree, exclude_files) print('Files removed:\n %s' % '\n '.join(pruned_files)) path_outzip = '%s.zip' % package_name zip_result_tree(path_outtree, path_outzip) print('Done.') return path_outzip # ------------------------------------------------------------------------------------------------- # CLI # ------------------------------------------------------------------------------------------------- def configure_argparser(): parser = ArgumentParser(description = 'Zip packaging tool for FreeRTOS release.') parser.add_argument('--core-input-zip', metavar = 'CORE-BASELINE.ZIP', default = None, help = 'FreeRTOS baseline zip to compare against new core zip') parser.add_argument('--labs-input-zip', metavar = 'LABS-BASELINE.ZIP', default = None, help = 'FreeRTOS-Labs baseline zip to compare agains new labs zip') parser.add_argument('--zip-version', metavar = 'PACKAGE_VERSION_NUMBER', type = str, default = None, help = 'Version number to be suffixed to FreeRTOS and FreeRTOS-Labs zips') parser.add_argument('--freertos-commit', metavar = 'FREERTOS_COMMIT_ID', type = str, default = 'HEAD', help = 'Commit ID of FreeRTOS repo to package') return parser def sanitize_cmd_args(args): # Check FreeRTOS Core options if not args.core_input_zip: info('No FreeRTOS baseline zip provided. Zip-comparison diagnostics will not be provided...') args.core_input_zip = None elif not os.path.exists(args.core_input_zip): error('Input zip does not exist: %s' % args.core_input_zip) exit(1) # Check FreeRTOS Labs options if not args.labs_input_zip: info('No FreeRTOS-Labs baseline zip provided. Zip-comparison diagnostics will not be provided...') args.labs_input_zip = None elif not os.path.exists(args.labs_input_zip): error('Input zip does not exist: %s' % args.input_zip) exit(1) # Check version options if args.zip_version == None: info('No version string provide. Will use "XX.YY.ZZ" as version suffix...') args.zip_version = 'XX.YY.ZZ' def main(): # CLI cmd = configure_argparser() # Setup args = cmd.parse_args() sanitize_cmd_args(args) setup_intermediate_files(DIR_INTERMEDIATE_FILES, DIR_INPUT_TREES, DIR_OUTPUT_TREES) # Create FreeRTOS and FreeRTOS-Labs packages core_package_name = 'FreeRTOSv%s' % args.zip_version (path_core_in_tree, path_core_out_tree) = create_file_trees(DIR_INPUT_TREES, args.core_input_zip, DIR_OUTPUT_TREES, FREERTOS_GIT_LINK, core_package_name, commit_id=args.freertos_commit) if path_core_out_tree == None: print('Failed to prepare repo for zipping') exit(1); core_outzip = create_package(path_core_out_tree, core_package_name, RELATIVE_FILE_EXCLUDES) # Create FreeRTOS-Labs package labs_package_name = 'FreeRTOS-Labs' (path_labs_in_tree, path_labs_out_tree) = create_file_trees(DIR_INPUT_TREES, args.labs_input_zip, DIR_OUTPUT_TREES, LABS_GIT_LINK, labs_package_name) if path_labs_out_tree == None: print('Failed to prepare repo for zipping') exit(1); labs_outzip = create_package(path_labs_out_tree, labs_package_name, LABS_RELATIVE_EXCLUDE_FILES) # Package summaries show_package_diagnostics(core_outzip, args.core_input_zip) show_package_diagnostics(labs_outzip, args.labs_input_zip) if __name__ == '__main__': main()