Select Git revision
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
winmake.py 33.13 KiB
"""
This tool is designed to facilitate downloading, patching, and building
of library dependencies for the Jami daemon project on windows. MSBuild
toolset and sdk versions can be supplied as parameters and will be sedded
into vcxproj files accordingly.
A package can be defined with a json like this:
{
"name": "mylibrary",
"version": "76a5006623539a58262d33458a5605be096b3a10",
"url": "https://git.example.com/gorblok/mylibrary/archive/__VERSION__.tar.gz",
"deps": ["mydep"],
"use_cmake" : true,
"defines": ["SEGFAULTS=0", "MY_CMAKE_DEFINE=true"],
"patches": ["some_patch.patch"],
"win_patches": ["some_windows_line_ending_patch.patch"],
"project_paths": ["mylibrary-static.vcxproj"],
"with_env" : "10.0.16299.0",
"custom_scripts": { "pre_build": [], "build": [], "post_build": [] }
}
"""
import sys
import os
import subprocess
import platform
import argparse
import json
import re
import zipfile
import tarfile
import multiprocessing
import shutil
import shlex
import glob
import time
from datetime import timedelta
import struct
import importlib
import logging
import traceback
import re
import fileinput
import hashlib
root_logger = logging.getLogger(__name__)
log = None
# project paths
daemon_msvc_dir = os.path.dirname(os.path.realpath(__file__))
daemon_dir = os.path.dirname(os.path.dirname(daemon_msvc_dir))
daemon_build_dir = daemon_dir + r'\build'
contrib_src_dir = daemon_dir + r'\contrib\src'
contrib_build_dir = daemon_dir + r'\contrib\build'
contrib_tmp_dir = daemon_dir + r'\contrib\tarballs'
plugins_bin_dir = daemon_dir + r'\..\plugins\build'
plugins_dir = daemon_dir + r'\..\plugins'
# SCM
wget_args = [
'--no-check-certificate', '--retry-connrefused',
'--waitretry=1', '--read-timeout=20',
'--timeout=15', '--tries=4']
git_apply_args = ['apply', '--reject',
'--ignore-whitespace', '--whitespace=fix']
patch_args = ['-flp1', '-i']
# vs help
win_sdk_default = '10.0.16299.0'
win_toolset_default = '142'
vs_where_path = os.path.join(
os.environ['ProgramFiles(x86)'], 'Microsoft Visual Studio', 'Installer', 'vswhere.exe'
)
host_is_64bit = (False, True)[platform.machine().endswith('64')]
python_is_64bit = (False, True)[8 * struct.calcsize("P") == 64]
def getMd5ForDirectory(path):
hasher = hashlib.md5()
for root, _, files in os.walk(path, topdown=True):
for name in files:
fileName = (os.path.join(root, name))
with open(str(fileName), 'rb') as aFile:
buf = aFile.read()
hasher.update(buf)
return hasher.hexdigest()
def shellquote(s, windows=False):
if not windows:
return "'" + s.replace("'", "'\''") + "'"
else:
return '\"' + s + '\"'
def getLatestVSVersion():
args = [
'-latest',
'-products *',
'-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64',
'-property installationVersion'
]
cmd = [vs_where_path] + args
output = subprocess.check_output(' '.join(cmd)).decode('utf-8')
if output:
return output.splitlines()[0].split('.')[0]
else:
return
def findVSLatestDir():
args = [
'-latest',
'-products *',
'-requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64',
'-property installationPath'
]
cmd = [vs_where_path] + args
output = subprocess.check_output(
' '.join(cmd)).decode(
'utf-8',
errors='ignore')
if output:
return output.splitlines()[0]
else:
return
def findMSBuild():
filename = 'MSBuild.exe'
for root, _, files in os.walk(findVSLatestDir() + r'\\MSBuild'):
if filename in files:
return os.path.join(root, filename)
def getVSEnv(arch='x64', platform='', version=''):
env_cmd = 'set path=%path:"=% && ' + \
getVSEnvCmd(arch, platform, version) + ' && set'
p = subprocess.Popen(env_cmd,
shell=True,
stdout=subprocess.PIPE)
stdout, _ = p.communicate()
out = stdout.decode('utf-8', errors='ignore').split("\r\n")[5:-1]
return dict(s.split('=', 1) for s in out)
def getCMakeGenerator(vs_version):
if vs_version == '15':
return '\"Visual Studio 15 2017 Win64\"'
else:
return '\"Visual Studio ' + vs_version + ' 2019\" -A x64'
def getVSEnvCmd(arch='x64', platform='', version=''):
vcEnvInit = [findVSLatestDir() + r'\VC\Auxiliary\Build\"vcvarsall.bat']
if platform != '':
args = [arch, platform, version]
else:
args = [arch, version]
if args:
vcEnvInit.extend(args)
vcEnvInit = 'call \"' + ' '.join(vcEnvInit)
return vcEnvInit
def make_plugin(pkg_info, force, sdk_version, toolset):
pkg_name = pkg_info.get('name')
version = pkg_info.get('version')
extractLibs = pkg_info.get('extractLibs')
plugin_path = (plugins_dir + "/" + pkg_name).replace("\\", "/")
if (extractLibs):
path = plugins_dir + r'\contrib\libs.tar.gz'
with tarfile.open(path, 'r', encoding="utf8", errors='ignore') as tarball:
tarball.extractall(plugins_dir + r'\contrib')
pkg_build_uptodate = False
pkg_ver_uptodate = False
# attempt to get the current built version
current_version = ''
# check build file for current version
build_file = plugins_bin_dir + r'\\.' + pkg_name
if os.path.exists(build_file):
if force:
os.remove(build_file)
else:
pkg_build_uptodate = is_build_uptodate(pkg_name, build_file)
with open(build_file, 'r+', encoding="utf8", errors='ignore') as f:
current_version = f.read()
if current_version == version:
pkg_ver_uptodate = True
for dep in pkg_info.get('deps', []):
dep_build_dep = resolve(dep, False, sdk_version, toolset)
if dep_build_dep:
pkg_build_uptodate = False
pkg_up_to_date = pkg_build_uptodate & pkg_ver_uptodate
cmake_defines = ""
for define in pkg_info.get('defines', []):
cmake_defines += " -D" + define + " "
if not pkg_up_to_date or current_version is None or force:
root_logger.warning(
"Building plugin with preferred sdk version %s and toolset %s", sdk_version, toolset)
env_set = 'false' if pkg_info.get('with_env', '') == '' else 'true'
sdk_to_use = sdk_version if env_set == 'false' else pkg_info.get(
'with_env', '')
cmake_script = "cmake -DCMAKE_SYSTEM_VERSION=" + sdk_version + \
" -G " + getCMakeGenerator(getLatestVSVersion()) + cmake_defines + \
" -T " + toolset + \
" -S " + plugin_path + " -B " + plugin_path + "/msvc"
root_logger.warning("Cmake generating vcxproj files")
_ = getSHrunner().exec_batch(cmake_script)
build(pkg_name,
plugin_path,
pkg_info.get('project_paths', []),
pkg_info.get('custom_scripts', {}),
env_set,
sdk_to_use,
toolset)
track_build(pkg_name, version, True)
def make_daemon(pkg_info, force, sdk_version, toolset):
cmake_script = 'cmake -DCMAKE_CONFIGURATION_TYPES="ReleaseLib_win32" -DCMAKE_SYSTEM_VERSION=' + sdk_version + \
' -DCMAKE_VS_PLATFORM_NAME="x64" -G ' + getCMakeGenerator(getLatestVSVersion(
)) + ' -T $(DefaultPlatformToolset) -S ../../ -B ../../build'
root_logger.warning("Cmake generating vcxproj files")
result = getSHrunner().exec_batch(cmake_script)
if result[0] != 0:
sys.exit("Cmake Errors")
for dep in pkg_info.get('deps', []):
resolve(dep, False, sdk_version, toolset)
root_logger.warning(
"Building daemon with preferred sdk version %s and toolset %s", sdk_version, toolset)
env_set = 'false' if pkg_info.get('with_env', '') == '' else 'true'
sdk_to_use = sdk_version if env_set == 'false' else pkg_info.get(
'with_env', '')
build('daemon', daemon_build_dir,
pkg_info.get('project_paths', []),
pkg_info.get('custom_scripts', {}),
env_set,
sdk_to_use,
toolset,
conf=pkg_info.get('configuration', 'Release'))
def make(pkg_info, force, sdk_version, toolset, isPlugin):
pkg_name = pkg_info.get('name')
if pkg_name == 'daemon':
return make_daemon(pkg_info, force, sdk_version, toolset)
if isPlugin:
return make_plugin(pkg_info, force, sdk_version, toolset)
md5 = getMd5ForDirectory(contrib_src_dir + r'\\' + pkg_name)
version = pkg_info.get('version')
pkg_build_uptodate = False
pkg_ver_uptodate = False
# attempt to get the current built version
current_version = ''
# check build file for current version
build_file = contrib_build_dir + r'\\.' + pkg_name
if os.path.exists(build_file):
if force:
os.remove(build_file)
else:
pkg_build_uptodate = is_build_uptodate(pkg_name, build_file)
with open(build_file, 'r+', encoding="utf8", errors='ignore') as f:
current_version = f.read()
if current_version == md5:
pkg_ver_uptodate = True
for dep in pkg_info.get('deps', []):
dep_build_dep = resolve(dep, False, sdk_version, toolset)
if dep_build_dep:
pkg_build_uptodate = False
pkg_up_to_date = pkg_build_uptodate & pkg_ver_uptodate
if not pkg_up_to_date or current_version is None or force:
if current_version != '':
log.debug(pkg_name + ' currently @: ' + current_version)
if force:
log.debug('Forcing fetch/patch/build for ' + pkg_name)
should_fetch = not pkg_up_to_date
pkg_build_path = contrib_build_dir + '\\' + pkg_name
if os.path.exists(pkg_build_path):
log.warning('Cleaning build for ' + pkg_name)
getSHrunner().exec_batch('rmdir', ['/s', '/q', pkg_build_path])
if not pkg_up_to_date or force:
if not force and not current_version is None:
log.warning(pkg_name + ' is not up to date')
if (should_fetch or force) and fetch_pkg(
pkg_name, version, pkg_info['url'], force):
apply(pkg_name, pkg_info.get('patches', []),
pkg_info.get('win_patches', []))
env_set = 'false' if pkg_info.get('with_env', '') == '' else 'true'
sdk_to_use = sdk_version if env_set == 'false' else pkg_info.get(
'with_env', '')
# configure with cmake ?
use_cmake = pkg_info.get('use_cmake', False)
if use_cmake:
cmake_defines = ""
for define in pkg_info.get('defines', []):
cmake_defines += " -D" + define
if not pkg_up_to_date or current_version is None or force:
cmake_conf_script = "cmake -G " + getCMakeGenerator(getLatestVSVersion(
)) + cmake_defines + " -S '" + pkg_build_path + "' -B '" + pkg_build_path + "\\build'"
log.debug("Configuring with Cmake")
result = getSHrunner().exec_batch(cmake_conf_script)
if result[0] != 0:
log.error("Error configuring with CMake")
exit(1)
if build(pkg_name,
contrib_build_dir + '\\' + pkg_name,
pkg_info.get('project_paths', []),
pkg_info.get('custom_scripts', {}),
env_set,
sdk_to_use,
toolset,
use_cmake=use_cmake):
track_build(pkg_name, md5, False)
else:
log.error("Couldn't build contrib " + pkg_name)
exit(1)
log.info(pkg_name + ' up to date')
return True
# did not need build
log.info(pkg_name + ' already up to date')
return False
def fetch_pkg(pkg_name, version, url, force):
version_replace = re.compile(re.escape('__VERSION__'))
full_url = version_replace.sub(version, url)
if not full_url:
log.error(pkg_name + ' missing url in package configuration')
return False
archive_name = full_url[full_url.rfind("/") + 1:]
archive_path = contrib_tmp_dir + '\\' + pkg_name + '_' + archive_name
if not os.path.exists(archive_path):
log.debug('Fetching ' + pkg_name + ' from: ' + full_url)
args = [full_url, '-O', archive_path]
args.extend(wget_args)
dl_result = getSHrunner().exec_batch('wget', args)
if dl_result[0] != 0:
log.warning(
'wget failure. Using powershell Invoke-WebRequest instead')
args = ['-Uri', full_url, '-OutFile', archive_path]
dl_result = getSHrunner().exec_ps1('Invoke-WebRequest', args)
return extract_archive(pkg_name, archive_name, archive_path)
else:
log.warning(archive_name +
' already exists in the tarball/archive directory')
decomp_result = extract_archive(pkg_name, archive_name, archive_path)
if not decomp_result and force:
log.debug('Removing old tarball for ' + archive_name)
getSHrunner().exec_batch('del', ['/s', '/q', archive_name])
return fetch_pkg(pkg_name, version, url, False)
else:
return True
return False
def remove_archive_if_needed(pkg_build_path, dirty_path):
if os.path.exists(pkg_build_path):
log.debug('Removing old package ' + pkg_build_path)
getSHrunner().exec_batch('rmdir', ['/s', '/q', pkg_build_path])
elif os.path.exists(dirty_path):
log.debug('Removing partial decompression ' + dirty_path)
getSHrunner().exec_batch('rmdir', ['/s', '/q', dirty_path])
def extract_tar(pkg_build_path, name, path):
with tarfile.open(path, 'r', encoding="utf8", errors='ignore') as tarball:
tar_common_prefix = os.path.commonprefix(tarball.getnames())
dirty_path = contrib_build_dir + '\\' + tar_common_prefix
remove_archive_if_needed(pkg_build_path, dirty_path)
log.debug('Decompressing ' + name + ' to ' + pkg_build_path)
tarball.extractall(contrib_build_dir)
os.rename(contrib_build_dir + '\\' + tar_common_prefix,
pkg_build_path)
return True
return False
def extract_zip(pkg_build_path, name, path):
with zipfile.ZipFile(path, 'r') as ziparchive:
zip_common_prefix = os.path.commonprefix(ziparchive.namelist())
dirty_path = contrib_build_dir + '\\' + zip_common_prefix
remove_archive_if_needed(pkg_build_path, dirty_path)
log.debug('Decompressing ' + name + ' to ' + pkg_build_path)
ziparchive.extractall(contrib_build_dir)
os.rename(contrib_build_dir + '\\' + zip_common_prefix,
pkg_build_path)
return True
return False
def extract_archive(pkg_name, name, path):
pkg_build_path = contrib_build_dir + '\\' + pkg_name
if tarfile.is_tarfile(path):
return extract_tar(pkg_build_path, name, path)
elif zipfile.is_zipfile(path):
return extract_zip(pkg_build_path, name, path)
def apply_linux(patch_path):
log.debug('applying linux patch ' + patch_path)
args = []
args.extend(patch_args)
args.append(patch_path)
return getSHrunner().exec_sh('patch', args)
def apply_windows(patch_path):
log.debug('applying windows patch ' + patch_path)
args = []
args.extend(git_apply_args)
args.append(patch_path)
return getSHrunner().exec_batch('git', args)
def apply(pkg_name, patches, win_patches):
log.debug('patching ' + pkg_name + '...')
tmp_dir = os.getcwd()
pkg_build_path = contrib_build_dir + '\\' + pkg_name
if not os.path.exists(pkg_build_path):
os.makedirs(pkg_build_path)
os.chdir(pkg_build_path)
base_sh_src_path = get_sh_path(contrib_src_dir)
# 1. git patches (LF)
for p in patches:
patch_path = base_sh_src_path + '/' + pkg_name + '/' + p
result = apply_linux(patch_path)
if result[0]:
log.error('Couldn\'t apply patch: ' + patch_path)
exit(1)
# 2. windows git patches (CR/LF)
for wp in win_patches:
patch_path = contrib_src_dir + '\\' + pkg_name + '\\' + wp
result = apply_windows(patch_path)
if result[0]:
log.error('Couldn\'t apply patch: ' + patch_path)
exit(1)
os.chdir(tmp_dir)
def get_pkg_file(pkg_name, isPlugin = False):
if pkg_name == 'daemon':
pkg_location = daemon_msvc_dir
elif (isPlugin):
pkg_location = plugins_dir + r'\\' + pkg_name
else:
pkg_location = daemon_dir + r'\contrib\src\\' + pkg_name
pkg_json_file = pkg_location + r"\\package.json"
if not os.path.exists(pkg_json_file):
log.error("No package info for " + pkg_name)
sys.exit(1)
return pkg_json_file
def resolve(pkg_name, force=False, sdk_version='', toolset='', isPlugin=False):
pkg_json_file = get_pkg_file(pkg_name, isPlugin)
with open(pkg_json_file, encoding="utf8", errors='ignore') as json_file:
log.info('Resolving: ' + pkg_name)
pkg_info = json.load(json_file)
try:
return make(pkg_info, force, sdk_version, toolset, isPlugin)
except Exception as e:
print(e)
log.error('Make ' + pkg_name + ' failed!')
sys.exit(1)
def track_build(pkg_name, version, isPlugin):
if isPlugin:
build_file = plugins_bin_dir + '\\.' + pkg_name
else:
build_file = contrib_build_dir + '\\.' + pkg_name
f = open(build_file, "w+", encoding="utf8", errors='ignore')
f.write(version)
f.close()
def build(pkg_name, pkg_dir, project_paths, custom_scripts, with_env, sdk,
toolset, arch='x64', conf='Release', use_cmake=False):
getMSbuilder().set_msbuild_configuration(with_env, arch, conf, toolset)
getMSbuilder().setup_vs_env(sdk)
success = True
build_operations = 0
tmp_dir = os.getcwd()
os.chdir(pkg_dir)
# pre_build custom step (CMake...)
pre_build_scripts = custom_scripts.get("pre_build", [])
if pre_build_scripts:
log.debug('Pre-build phase')
for script in pre_build_scripts:
result = getSHrunner().exec_batch(script)
success &= not result[0]
build_operations += 1
# build custom step (nmake...)
build_scripts = custom_scripts.get("build", [])
if build_scripts:
log.debug('Custom Build phase')
for script in build_scripts:
result = getSHrunner().exec_batch(script)
success &= not result[0]
build_operations += 1
# vcxproj files
if project_paths:
log.debug('Msbuild phase')
for pp in project_paths:
project_full_path = pkg_dir + '\\' + pp
log.debug('Building: ' + pkg_name + " with sdk version " +
sdk + " and toolset " + toolset)
getMSbuilder().build(pkg_name, project_full_path, sdk, toolset)
build_operations += 1
else:
# build directly with cmake
if use_cmake is True:
log.debug('CMake build phase')
cmake_build_script = "cmake --build '" + pkg_dir + \
"\\build' " + "--config " + conf
result = getSHrunner().exec_batch(cmake_build_script)
if result[0] != 0:
log.error("Error building with CMake")
exit(1)
post_build_scripts = custom_scripts.get("post_build", [])
if post_build_scripts:
log.debug('Post-Build phase')
for script in post_build_scripts:
result = getSHrunner().exec_batch(script)
success &= not result[0]
build_operations += 1
os.chdir(tmp_dir)
# should cover header only, no cmake, etc
ops = len(build_scripts) + len(project_paths) + \
len(pre_build_scripts) + len(post_build_scripts)
return success and build_operations == ops
class Singleton:
def __init__(self, decorated):
self._decorated = decorated
def instance(self):
try:
return self._instance
except AttributeError:
self._instance = self._decorated()
return self._instance
def __call__(self):
raise TypeError('Singletons must be accessed through `instance()`.')
def __instancecheck__(self, inst):
return isinstance(inst, self._decorated)
class ScriptType:
ps1 = 1
cmd = 2
sh = 3
@Singleton
class SHrunner():
def __init__(self):
sys_path = (r'\Sysnative', r'\system32')[python_is_64bit]
full_sys_path = os.path.expandvars('%systemroot%') + sys_path
# powershell
self.ps_path = full_sys_path + r'\WindowsPowerShell\v1.0\powershell.exe'
if not os.path.exists(self.ps_path):
log.error('Powershell not found at %s.' % self.ps_path)
sys.exit(1)
# bash
if not os.environ.get('JENKINS_URL'):
self.sh_path = full_sys_path + r'\bash.exe'
else:
self.sh_path = r'\"C:\Program Files\Git\git-bash.exe\"'
if not os.path.exists(self.sh_path):
log.warning('Bash not found at ' + self.sh_path)
self.sh_path = shutil.which('bash.exe')
if not os.path.exists(self.sh_path):
log.error('No bash found')
sys.exit(1)
else:
self.sh_path = shellquote(self.sh_path, windows=True)
log.debug('Using alternate bash found at ' + self.sh_path)
self.project_env_vars = {
'DAEMON_DIR': daemon_dir,
'CONTRIB_SRC_DIR': contrib_src_dir,
'CONTRIB_BUILD_DIR': contrib_build_dir,
'VCVARSALL_CMD': getVSEnvCmd(),
'CMAKE_GENERATOR': getCMakeGenerator(getLatestVSVersion())
}
self.base_env_vars = self.project_env_vars.copy()
self.base_env_vars.update(os.environ.copy())
self.vs_env_vars = {}
def set_vs_env_vars(self, env_target):
self.vs_env_vars = {}
self.vs_env_vars = self.project_env_vars.copy()
self.vs_env_vars.update(getVSEnv(version=env_target))
def exec_script(self, script_type=ScriptType.cmd, script=None, args=[]):
if script_type is ScriptType.cmd:
cmd = [script]
if not args:
cmd = shlex.split(script)
elif script_type is ScriptType.ps1:
cmd = [self.ps_path, '-ExecutionPolicy', 'ByPass', script]
elif script_type is ScriptType.sh:
cmd = [self.sh_path, '-c ', '\"' + script]
if args:
cmd.extend(args)
if script_type is ScriptType.sh:
cmd[-1] = cmd[-1] + '\"'
cmd = " ".join(cmd)
p = subprocess.Popen(cmd,
shell=True,
stderr=sys.stderr,
stdout=sys.stdout,
env=self.vs_env_vars if self.vs_env_vars else self.base_env_vars)
rtrn, perr = p.communicate()
rcode = p.returncode
data = None
if perr:
data = json.dumps(perr.decode('utf-8', 'ignore'))
else:
data = rtrn
return rcode, data
def exec_batch(self, script=None, args=[]):
return self.exec_script(ScriptType.cmd, script, args)
def exec_ps1(self, script=None, args=[]):
return self.exec_script(ScriptType.ps1, script, args)
def exec_sh(self, script=None, args=[]):
return self.exec_script(ScriptType.sh, script, args)
def getSHrunner():
return SHrunner.instance()
@Singleton
class MSbuilder:
def __init__(self):
self.vsenv_done = False
self.msbuild = findMSBuild()
self.default_msbuild_args = [
'/nologo',
'/verbosity:normal',
'/maxcpucount:' + str(multiprocessing.cpu_count())]
self.set_msbuild_configuration()
def set_msbuild_configuration(self, with_env='false', arch='x64',
configuration='Release',
toolset=win_toolset_default):
self.extra_msbuild_args = [
'/p:Platform=' + arch,
'/p:Configuration=' + configuration,
'/p:PlatformToolset=' + toolset,
'/p:useenv=' + with_env
]
@staticmethod
def replace_vs_prop(filename, prop, val):
p = re.compile(r'(?s)<' + prop + r'\s?.*?>(.*?)<\/' + prop + r'>')
val = r'<' + prop + r'>' + val + r'</' + prop + r'>'
with fileinput.FileInput(filename, inplace=True) as file:
for line in file:
print(re.sub(p, val, line), end='')
def build(self, pkg_name, proj_path, sdk_version, toolset):
if not os.path.isfile(self.msbuild):
raise IOError('msbuild.exe not found. path=' + self.msbuild)
if os.environ.get('JENKINS_URL'):
log.info("Jenkins Clear DebugInformationFormat")
self.__class__.replace_vs_prop(proj_path,
'DebugInformationFormat',
'None')
# force chosen sdk
self.__class__.replace_vs_prop(proj_path,
'WindowsTargetPlatformVersion',
sdk_version)
# force chosen toolset
self.__class__.replace_vs_prop(proj_path,
'PlatformToolset',
toolset)
args = []
args.extend(self.default_msbuild_args)
args.extend(self.extra_msbuild_args)
args.append(proj_path)
result = getSHrunner().exec_batch(self.msbuild, args)
if result[0] == 1:
log.error("Build failed when building " + pkg_name)
sys.exit(1)
def setup_vs_env(self, env_target):
if self.vsenv_done:
log.debug('vs environment already initialized')
return
log.debug('Setting up vs environment')
getSHrunner().set_vs_env_vars(env_target)
self.vsenv_done = True
def getMSbuilder():
return MSbuilder.instance()
def parse_args():
ap = argparse.ArgumentParser(description="Windows Jami build tool")
ap.add_argument(
'-b', '--build',
help='Build latest contrib')
ap.add_argument(
'-f', '--force', action='store_true',
help='Force action')
ap.add_argument(
'-c', '--clean',
help='Cleans out build directory for a contrib')
ap.add_argument(
'-d', '--debug', default='DEBUG',
help='Sets the logging level')
ap.add_argument(
'-i', '--indent', action='store_true',
help='Sets whether the logs are indented to stack frames')
ap.add_argument(
'-v', '--verbose', action='store_true',
help='Sets whether the logs are verbose or not')
ap.add_argument(
'-p', '--purge', action='store_true',
help='Cleans out contrib tarball directory')
ap.add_argument(
'-s', '--sdk', default=win_sdk_default, type=str,
help='Use specified windows sdk version')
ap.add_argument(
'-t', '--toolset', default=win_toolset_default, type=str,
help='Use specified platform toolset version')
ap.add_argument(
'-P', '--plugin', default=False, action='store_true',
help="Defines if we're building a plugin"
)
parsed_args = ap.parse_args()
if parsed_args.toolset:
if parsed_args.toolset[0] != 'v':
parsed_args.toolset = 'v' + parsed_args.toolset
return parsed_args
def main():
start_time = time.time()
parsed_args = parse_args()
setup_logging(lvl=parsed_args.debug,
verbose=parsed_args.verbose,
do_indent=parsed_args.indent)
if not host_is_64bit:
log.error('These scripts will only run on a 64-bit Windows system for now!')
sys.exit(1)
if int(getLatestVSVersion()) < 15:
log.error('These scripts require at least Visual Studio v15 2017!')
sys.exit(1)
if parsed_args.purge:
if os.path.exists(contrib_tmp_dir):
log.warning('Removing contrib tarballs ' + contrib_tmp_dir)
getSHrunner().exec_batch(
'del', ['/s', '/f', '/q', contrib_tmp_dir + '\\*.tar.*'])
getSHrunner().exec_batch(
'del', ['/s', '/f', '/q', contrib_tmp_dir + '\\*.tgz.*'])
getSHrunner().exec_batch(
'del', ['/s', '/f', '/q', contrib_tmp_dir + '\\*.zip.*'])
else:
log.warning('No tarballs to remove')
if parsed_args.clean:
if os.path.exists(contrib_build_dir) and parsed_args.clean == 'all':
log.warning('Removing contrib builds ' + contrib_build_dir)
getSHrunner().exec_batch('rmdir', ['/s', '/q', contrib_build_dir])
else:
pkg_json_file = get_pkg_file(parsed_args.clean)
with open(pkg_json_file, encoding="utf8", errors='ignore') as json_file:
pkg_info = json.load(json_file)
if (not parsed_args.plugin):
exclude_dirs = contrib_build_dir
dir_to_clean = exclude_dirs + '\\' + pkg_info['name']
file_to_clean = exclude_dirs + '\\.' + pkg_info['name']
if os.path.exists(dir_to_clean) or os.path.exists(
file_to_clean):
log.warning('Removing build ' + dir_to_clean)
getSHrunner().exec_batch(
'rmdir', ['/s', '/q', dir_to_clean])
getSHrunner().exec_batch(
'del', ['/s', '/f', '/q', file_to_clean])
else:
log.warning('No builds to remove')
else:
exclude_dirs = plugins_bin_dir
files_to_clean = [exclude_dirs + '\\.' + pkg_info['name']]
for name in glob.iglob(
exclude_dirs + '\\**\\' + pkg_info['name'] + ".jpl", recursive=True):
files_to_clean.append(name)
for name in files_to_clean:
getSHrunner().exec_batch(
'del', ['/s', '/f', '/q', name])
getSHrunner().exec_batch(
'rmdir', ['/s', '/q', plugins_dir + "\\" + parsed_args.clean + "\\msvc"])
if parsed_args.build:
if not os.path.exists(contrib_build_dir):
os.makedirs(contrib_build_dir)
log.info('Making: ' + parsed_args.build)
resolve(parsed_args.build, parsed_args.force,
parsed_args.sdk, parsed_args.toolset, parsed_args.plugin)
log.info('Make done for: ' + parsed_args.build)
log.debug("--- %s ---" % secondsToStr(time.time() - start_time))
def get_sh_path(path):
driveless_path = path.replace(os.path.sep, '/')[3:]
drive_letter = os.path.splitdrive(daemon_dir)[0][0].lower()
wsl_drive_path = '/mnt/' + drive_letter + '/'
no_echo = ' &> /dev/null'
result = getSHrunner().exec_sh('pwd | grep ' + wsl_drive_path + no_echo)
if result[0]:
# using git bash
return '/' + drive_letter + '/' + driveless_path
# using wsl
return wsl_drive_path + driveless_path
def newest_file(root):
file_list = []
for path, _, files in os.walk(root):
for name in files:
file_list.append(os.path.join(path, name))
latest_file = max(file_list, key=os.path.getmtime)
return latest_file
def is_build_uptodate(pkg_name, build_file):
root = contrib_build_dir + '\\' + pkg_name
file_list = []
for path, _, files in os.walk(root):
for name in files:
file_list.append(os.path.join(path, name))
if not file_list:
return False
latest_file = max(file_list, key=os.path.getmtime)
t_mod = os.path.getmtime(latest_file)
t_build = os.path.getmtime(build_file)
return t_mod < t_build
def secondsToStr(elapsed=None):
return str(timedelta(seconds=elapsed))
class CustomAdapter(logging.LoggerAdapter):
@staticmethod
def indent():
indentation_level = len(traceback.extract_stack())
return indentation_level - 4 - 2 # Remove logging infrastructure frames
def process(self, msg, kwargs):
return '{i}{m}'.format(i=' ' * (self.indent()), m=msg), kwargs
def setup_logging(lvl=logging.DEBUG, verbose=False, do_indent=False):
format = ''
if verbose:
format = '[ %(levelname)-8s %(created).6f %(funcName)10s:%(lineno)4s ] '
fmt = format + '%(message)s'
try:
import coloredlogs
coloredlogs.install(
level=lvl,
logger=root_logger,
fmt=fmt,
level_styles={
'debug': {'color': 'blue'},
'info': {'color': 'green'},
'warn': {'color': 'yellow'},
'error': {'color': 'red'},
'critical': {'color': 'red', 'bold': True}
},
field_styles={
'asctime': {'color': 'magenta'},
'created': {'color': 'magenta'},
'levelname': {'color': 'cyan'},
'funcName': {'color': 'black', 'bold': True},
'lineno': {'color': 'black', 'bold': True}
})
except ImportError:
root_logger.setLevel(logging.DEBUG)
logging.basicConfig(level=lvl, format=fmt)
if do_indent:
global log
log = CustomAdapter(logging.getLogger(__name__), {})
else:
log = logging.getLogger(__name__)
if __name__ == '__main__':
main()