Finalize Azure Pipelines Definitions

d
This commit is contained in:
Zach Hilman 2019-07-13 21:34:40 -04:00
parent bb67091c77
commit bbc5b5d62d
29 changed files with 572 additions and 35 deletions

View File

View File

@ -0,0 +1,15 @@
#!/bin/bash -ex
# Copy documentation
cp license.txt "$REV_NAME"
cp README.md "$REV_NAME"
tar $COMPRESSION_FLAGS "$ARCHIVE_NAME" "$REV_NAME"
mv "$REV_NAME" $RELEASE_NAME
7z a "$REV_NAME.7z" $RELEASE_NAME
# move the compiled archive into the artifacts directory to be uploaded by travis releases
mv "$ARCHIVE_NAME" artifacts/
mv "$REV_NAME.7z" artifacts/

View File

@ -0,0 +1,6 @@
#!/bin/bash -ex
GITDATE="`git show -s --date=short --format='%ad' | sed 's/-//g'`"
GITREV="`git show -s --format='%h'`"
mkdir -p artifacts

View File

@ -0,0 +1,6 @@
#!/bin/bash -ex
# Run clang-format
cd /yuzu
chmod a+x ./.ci/scripts/format/script.sh
./.ci/scripts/format/script.sh

View File

@ -0,0 +1,4 @@
#!/bin/bash -ex
chmod a+x ./.ci/scripts/format/docker.sh
docker run -v $(pwd):/yuzu yuzuemu/build-environments:linux-clang-format /bin/bash -ex /yuzu/.ci/scripts/format/docker.sh

View File

@ -0,0 +1,37 @@
#!/bin/bash -ex
if grep -nrI '\s$' src *.yml *.txt *.md Doxyfile .gitignore .gitmodules .ci* dist/*.desktop \
dist/*.svg dist/*.xml; then
echo Trailing whitespace found, aborting
exit 1
fi
# Default clang-format points to default 3.5 version one
CLANG_FORMAT=clang-format-6.0
$CLANG_FORMAT --version
if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then
# Get list of every file modified in this pull request
files_to_lint="$(git diff --name-only --diff-filter=ACMRTUXB $TRAVIS_COMMIT_RANGE | grep '^src/[^.]*[.]\(cpp\|h\)$' || true)"
else
# Check everything for branch pushes
files_to_lint="$(find src/ -name '*.cpp' -or -name '*.h')"
fi
# Turn off tracing for this because it's too verbose
set +x
for f in $files_to_lint; do
d=$(diff -u "$f" <($CLANG_FORMAT "$f") || true)
if ! [ -z "$d" ]; then
echo "!!! $f not compliant to coding style, here is the fix:"
echo "$d"
fail=1
fi
done
set -x
if [ "$fail" = 1 ]; then
exit 1
fi

View File

@ -0,0 +1,14 @@
#!/bin/bash -ex
cd /yuzu
ccache -s
mkdir build || true && cd build
cmake .. -G Ninja -DYUZU_USE_BUNDLED_UNICORN=ON -DYUZU_USE_QT_WEB_ENGINE=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=/usr/lib/ccache/gcc -DCMAKE_CXX_COMPILER=/usr/lib/ccache/g++ -DYUZU_ENABLE_COMPATIBILITY_REPORTING=${ENABLE_COMPATIBILITY_REPORTING:-"OFF"} -DENABLE_COMPATIBILITY_LIST_DOWNLOAD=ON -DUSE_DISCORD_PRESENCE=ON
ninja
ccache -s
ctest -VV -C Release

View File

@ -0,0 +1,5 @@
#!/bin/bash -ex
mkdir -p "ccache" || true
chmod a+x ./.ci/scripts/linux/docker.sh
docker run -e ENABLE_COMPATIBILITY_REPORTING -e CCACHE_DIR=/yuzu/ccache -v $(pwd):/yuzu yuzuemu/build-environments:linux-fresh /bin/bash /yuzu/.ci/scripts/linux/docker.sh

View File

@ -0,0 +1,14 @@
#!/bin/bash -ex
. .ci/scripts/common/pre-upload.sh
REV_NAME="yuzu-linux-${GITDATE}-${GITREV}"
ARCHIVE_NAME="${REV_NAME}.tar.xz"
COMPRESSION_FLAGS="-cJvf"
mkdir "$REV_NAME"
cp build/bin/yuzu-cmd "$REV_NAME"
cp build/bin/yuzu "$REV_NAME"
. .ci/scripts/common/post-upload.sh

View File

@ -0,0 +1,28 @@
# Download all pull requests as patches that match a specific label
# Usage: python download-patches-by-label.py <Label to Match> <Root Path Folder to DL to>
import requests, sys, json, urllib3.request, shutil, subprocess
http = urllib3.PoolManager()
dl_list = {}
def check_individual(labels):
for label in labels:
if (label["name"] == sys.argv[1]):
return True
return False
try:
url = 'https://api.github.com/repos/yuzu-emu/yuzu/pulls'
response = requests.get(url)
if (response.ok):
j = json.loads(response.content)
for pr in j:
if (check_individual(pr["labels"])):
pn = pr["number"]
print("Matched PR# %s" % pn)
print(subprocess.check_output(["git", "fetch", "https://github.com/yuzu-emu/yuzu.git", "pull/%s/head:pr-%s" % (pn, pn), "-f"]))
print(subprocess.check_output(["git", "merge", "--squash", "pr-%s" % pn]))
print(subprocess.check_output(["git", "commit", "-m\"Merge PR %s\"" % pn]))
except:
sys.exit(-1)

View File

@ -0,0 +1,18 @@
# Checks to see if the specified pull request # has the specified tag
# Usage: python check-label-presence.py <Pull Request ID> <Name of Label>
import requests, json, sys
try:
url = 'https://api.github.com/repos/yuzu-emu/yuzu/issues/%s' % sys.argv[1]
response = requests.get(url)
if (response.ok):
j = json.loads(response.content)
for label in j["labels"]:
if label["name"] == sys.argv[2]:
print('##vso[task.setvariable variable=enabletesting;]true')
sys.exit()
except:
sys.exit(-1)
print('##vso[task.setvariable variable=enabletesting;]false')

View File

@ -0,0 +1,2 @@
git config --global user.email "yuzu@yuzu-emu.org"
git config --global user.name "yuzubot"

View File

@ -0,0 +1,50 @@
#!/bin/bash -ex
cd /yuzu
ccache -s
# Dirty hack to trick unicorn makefile into believing we are in a MINGW system
mv /bin/uname /bin/uname1 && echo -e '#!/bin/sh\necho MINGW64' >> /bin/uname
chmod +x /bin/uname
# Dirty hack to trick unicorn makefile into believing we have cmd
echo '' >> /bin/cmd
chmod +x /bin/cmd
mkdir build || true && cd build
cmake .. -G Ninja -DCMAKE_TOOLCHAIN_FILE="$(pwd)/../CMakeModules/MinGWCross.cmake" -DUSE_CCACHE=ON -DYUZU_USE_BUNDLED_UNICORN=ON -DENABLE_COMPATIBILITY_LIST_DOWNLOAD=ON -DCMAKE_BUILD_TYPE=Release
ninja
# Clean up the dirty hacks
rm /bin/uname && mv /bin/uname1 /bin/uname
rm /bin/cmd
ccache -s
echo "Tests skipped"
#ctest -VV -C Release
echo 'Prepare binaries...'
cd ..
mkdir package
QT_PLATFORM_DLL_PATH='/usr/x86_64-w64-mingw32/lib/qt5/plugins/platforms/'
find build/ -name "yuzu*.exe" -exec cp {} 'package' \;
# copy Qt plugins
mkdir package/platforms
cp "${QT_PLATFORM_DLL_PATH}/qwindows.dll" package/platforms/
cp -rv "${QT_PLATFORM_DLL_PATH}/../mediaservice/" package/
cp -rv "${QT_PLATFORM_DLL_PATH}/../imageformats/" package/
rm -f package/mediaservice/*d.dll
for i in package/*.exe; do
# we need to process pdb here, however, cv2pdb
# does not work here, so we just simply strip all the debug symbols
x86_64-w64-mingw32-strip "${i}"
done
pip3 install pefile
python3 .ci/scripts/windows/scan_dll.py package/*.exe "package/"
python3 .ci/scripts/windows/scan_dll.py package/imageformats/*.dll "package/"

View File

@ -0,0 +1,5 @@
#!/bin/bash -ex
mkdir -p "ccache" || true
chmod a+x ./.ci/scripts/windows/docker.sh
docker run -e CCACHE_DIR=/yuzu/ccache -v $(pwd):/yuzu yuzuemu/build-environments:linux-mingw /bin/bash -ex /yuzu/.ci/scripts/windows/docker.sh

View File

@ -0,0 +1,106 @@
import pefile
import sys
import re
import os
import queue
import shutil
# constant definitions
KNOWN_SYS_DLLS = ['WINMM.DLL', 'MSVCRT.DLL', 'VERSION.DLL', 'MPR.DLL',
'DWMAPI.DLL', 'UXTHEME.DLL', 'DNSAPI.DLL', 'IPHLPAPI.DLL']
# below is for Ubuntu 18.04 with specified PPA enabled, if you are using
# other distro or different repositories, change the following accordingly
DLL_PATH = [
'/usr/x86_64-w64-mingw32/bin/',
'/usr/x86_64-w64-mingw32/lib/',
'/usr/lib/gcc/x86_64-w64-mingw32/7.3-posix/'
]
missing = []
def parse_imports(file_name):
results = []
pe = pefile.PE(file_name, fast_load=True)
pe.parse_data_directories()
for entry in pe.DIRECTORY_ENTRY_IMPORT:
current = entry.dll.decode()
current_u = current.upper() # b/c Windows is often case insensitive
# here we filter out system dlls
# dll w/ names like *32.dll are likely to be system dlls
if current_u.upper() not in KNOWN_SYS_DLLS and not re.match(string=current_u, pattern=r'.*32\.DLL'):
results.append(current)
return results
def parse_imports_recursive(file_name, path_list=[]):
q = queue.Queue() # create a FIFO queue
# file_name can be a string or a list for the convience
if isinstance(file_name, str):
q.put(file_name)
elif isinstance(file_name, list):
for i in file_name:
q.put(i)
full_list = []
while q.qsize():
current = q.get_nowait()
print('> %s' % current)
deps = parse_imports(current)
# if this dll does not have any import, ignore it
if not deps:
continue
for dep in deps:
# the dependency already included in the list, skip
if dep in full_list:
continue
# find the requested dll in the provided paths
full_path = find_dll(dep)
if not full_path:
missing.append(dep)
continue
full_list.append(dep)
q.put(full_path)
path_list.append(full_path)
return full_list
def find_dll(name):
for path in DLL_PATH:
for root, _, files in os.walk(path):
for f in files:
if name.lower() == f.lower():
return os.path.join(root, f)
def deploy(name, dst, dry_run=False):
dlls_path = []
parse_imports_recursive(name, dlls_path)
for dll_entry in dlls_path:
if not dry_run:
shutil.copy(dll_entry, dst)
else:
print('[Dry-Run] Copy %s to %s' % (dll_entry, dst))
print('Deploy completed.')
return dlls_path
def main():
if len(sys.argv) < 3:
print('Usage: %s [files to examine ...] [target deploy directory]')
return 1
to_deploy = sys.argv[1:-1]
tgt_dir = sys.argv[-1]
if not os.path.isdir(tgt_dir):
print('%s is not a directory.' % tgt_dir)
return 1
print('Scanning dependencies...')
deploy(to_deploy, tgt_dir)
if missing:
print('Following DLLs are not found: %s' % ('\n'.join(missing)))
return 0
if __name__ == '__main__':
main()

View File

@ -0,0 +1,13 @@
#!/bin/bash -ex
. .ci/scripts/common/pre-upload.sh
REV_NAME="yuzu-windows-mingw-${GITDATE}-${GITREV}"
ARCHIVE_NAME="${REV_NAME}.tar.gz"
COMPRESSION_FLAGS="-czvf"
mkdir "$REV_NAME"
# get around the permission issues
cp -r package/* "$REV_NAME"
. .ci/scripts/common/post-upload.sh

View File

@ -0,0 +1,21 @@
parameters:
artifactSource: 'true'
steps:
- task: DockerInstaller@0
displayName: 'Prepare Environment'
inputs:
dockerVersion: '17.09.0-ce'
- task: CacheBeta@0
displayName: 'Cache Build System'
inputs:
key: yuzu-v1-$(BuildName)-$(BuildSuffix)-$(CacheSuffix)
path: $(System.DefaultWorkingDirectory)/ccache
cacheHitVar: CACHE_RESTORED
- script: chmod a+x ./.ci/scripts/$(ScriptFolder)/exec.sh && ./.ci/scripts/$(ScriptFolder)/exec.sh
displayName: 'Build'
- script: chmod a+x ./.ci/scripts/$(ScriptFolder)/upload.sh && ./.ci/scripts/$(ScriptFolder)/upload.sh
displayName: 'Package Artifacts'
- publish: artifacts
artifact: 'yuzu-$(BuildName)-$(BuildSuffix)'
displayName: 'Upload Artifacts'

View File

@ -0,0 +1,22 @@
jobs:
- job: build
displayName: 'standard'
pool:
vmImage: ubuntu-latest
strategy:
maxParallel: 10
matrix:
windows:
BuildSuffix: 'windows-mingw'
ScriptFolder: 'windows'
linux:
BuildSuffix: 'linux'
ScriptFolder: 'linux'
steps:
- template: ./sync-source.yml
parameters:
artifactSource: $(parameters.artifactSource)
needSubmodules: 'true'
- template: ./build-single.yml
parameters:
artifactSource: 'false'

View File

@ -0,0 +1,30 @@
jobs:
- job: build_test
displayName: 'testing'
pool:
vmImage: ubuntu-latest
strategy:
maxParallel: 10
matrix:
windows:
BuildSuffix: 'windows-testing'
ScriptFolder: 'windows'
steps:
- task: PythonScript@0
condition: eq(variables['Build.Reason'], 'PullRequest')
displayName: 'Determine Testing Status'
inputs:
scriptSource: 'filePath'
scriptPath: '../scripts/merge/check-label-presence.py'
arguments: '$(System.PullRequest.PullRequestNumber) create-testing-build'
- ${{ if eq(variables.enabletesting, 'true') }}:
- template: ./sync-source.yml
parameters:
artifactSource: $(parameters.artifactSource)
needSubmodules: 'true'
- template: ./mergebot.yml
parameters:
matchLabel: 'testing-merge'
- template: ./build-single.yml
parameters:
artifactSource: 'false'

View File

@ -0,0 +1,14 @@
parameters:
artifactSource: 'true'
steps:
- template: ./sync-source.yml
parameters:
artifactSource: $(parameters.artifactSource)
needSubmodules: 'false'
- task: DockerInstaller@0
displayName: 'Prepare Environment'
inputs:
dockerVersion: '17.09.0-ce'
- script: chmod a+x ./.ci/scripts/format/exec.sh && ./.ci/scripts/format/exec.sh
displayName: 'Verify Formatting'

46
.ci/templates/merge.yml Normal file
View File

@ -0,0 +1,46 @@
jobs:
- job: merge
displayName: 'pull requests'
steps:
- checkout: self
submodules: recursive
- template: ./mergebot.yml
parameters:
matchLabel: '$(BuildName)-merge'
- task: ArchiveFiles@2
displayName: 'Package Source'
inputs:
rootFolderOrFile: '$(System.DefaultWorkingDirectory)'
includeRootFolder: false
archiveType: '7z'
archiveFile: '$(Build.ArtifactStagingDirectory)/yuzu-$(BuildName)-source.7z'
- task: PublishPipelineArtifact@1
displayName: 'Upload Artifacts'
inputs:
targetPath: '$(Build.ArtifactStagingDirectory)/yuzu-$(BuildName)-source.7z'
artifact: 'yuzu-$(BuildName)-source'
replaceExistingArchive: true
- job: upload_source
displayName: 'upload'
dependsOn: merge
steps:
- template: ./sync-source.yml
parameters:
artifactSource: 'true'
needSubmodules: 'true'
- script: chmod a+x $(System.DefaultWorkingDirectory)/.ci/scripts/merge/yuzubot-git-config.sh && $(System.DefaultWorkingDirectory)/.ci/scripts/merge/yuzubot-git-config.sh
displayName: 'Apply Git Configuration'
- script: git tag -a $(BuildName)-$(Build.BuildId) -m "yuzu $(BuildName) $(Build.BuildNumber) $(Build.DefinitionName)"
displayName: 'Tag Source'
- script: git remote add other $(GitRepoPushChangesURL)
displayName: 'Register Repository'
- script: git push --follow-tags --force other HEAD:$(GitPushBranch)
displayName: 'Update Code'
- script: git rev-list -n 1 $(BuildName)-$(Build.BuildId) > $(Build.ArtifactStagingDirectory)/tag-commit.sha
displayName: 'Calculate Release Point'
- task: PublishPipelineArtifact@1
displayName: 'Upload Release Point'
inputs:
targetPath: '$(Build.ArtifactStagingDirectory)/tag-commit.sha'
artifact: 'yuzu-$(BuildName)-release-point'
replaceExistingArchive: true

View File

@ -0,0 +1,15 @@
parameters:
matchLabel: 'dummy-merge'
steps:
- script: mkdir $(System.DefaultWorkingDirectory)/patches && pip install requests urllib3
displayName: 'Prepare Environment'
- script: chmod a+x $(System.DefaultWorkingDirectory)/.ci/scripts/merge/yuzubot-git-config.sh && $(System.DefaultWorkingDirectory)/.ci/scripts/merge/yuzubot-git-config.sh
displayName: 'Apply Git Configuration'
- task: PythonScript@0
displayName: 'Discover, Download, and Apply Patches'
inputs:
scriptSource: 'filePath'
scriptPath: '.ci/scripts/merge/apply-patches-by-label.py'
arguments: '${{ parameters.matchLabel }} patches'
workingDirectory: '$(System.DefaultWorkingDirectory)'

29
.ci/templates/release.yml Normal file
View File

@ -0,0 +1,29 @@
steps:
- task: DownloadPipelineArtifact@2
displayName: 'Download Windows Release'
inputs:
artifactName: 'yuzu-$(BuildName)-windows-mingw'
buildType: 'current'
targetPath: '$(Build.ArtifactStagingDirectory)'
- task: DownloadPipelineArtifact@2
displayName: 'Download Linux Release'
inputs:
artifactName: 'yuzu-$(BuildName)-linux'
buildType: 'current'
targetPath: '$(Build.ArtifactStagingDirectory)'
- task: DownloadPipelineArtifact@2
displayName: 'Download Release Point'
inputs:
artifactName: 'yuzu-$(BuildName)-release-point'
buildType: 'current'
targetPath: '$(Build.ArtifactStagingDirectory)'
- script: echo '##vso[task.setvariable variable=tagcommit]' && cat $(Build.ArtifactStagingDirectory)/tag-commit.sha
displayName: 'Calculate Release Point'
- task: GitHubRelease@0
inputs:
gitHubConnection: $(GitHubReleaseConnectionName)
repositoryName: '$(GitHubReleaseRepoName)'
action: 'create'
target: $(variables.tagcommit)
title: 'yuzu $(BuildName) #$(Build.BuildId)'
assets: '$(Build.ArtifactStagingDirectory)/*'

View File

@ -0,0 +1,16 @@
steps:
- checkout: none
- task: DownloadPipelineArtifact@2
displayName: 'Download Source'
inputs:
artifactName: 'yuzu-$(BuildName)-source'
buildType: 'current'
targetPath: '$(Build.ArtifactStagingDirectory)'
- script: rm -rf $(System.DefaultWorkingDirectory) && mkdir $(System.DefaultWorkingDirectory)
displayName: 'Clean Working Directory'
- task: ExtractFiles@1
displayName: 'Prepare Source'
inputs:
archiveFilePatterns: '$(Build.ArtifactStagingDirectory)/*.7z'
destinationFolder: '$(System.DefaultWorkingDirectory)'
cleanDestinationFolder: false

View File

@ -0,0 +1,11 @@
parameters:
needSubmodules: 'true'
steps:
- checkout: self
displayName: 'Checkout Recursive'
submodules: recursive
# condition: eq(parameters.needSubmodules, 'true')
#- checkout: self
# displayName: 'Checkout Fast'
# condition: ne(parameters.needSubmodules, 'true')

View File

@ -0,0 +1,7 @@
steps:
- ${{ if eq(parameters.artifactSource, 'true') }}:
- template: ./retrieve-artifact-source.yml
- ${{ if ne(parameters.artifactSource, 'true') }}:
- template: ./retrieve-master-source.yml
parameters:
needSubmodules: $(parameters.needSubmodules)

View File

@ -1,19 +1,23 @@
# Starter pipeline
# Start with a minimal pipeline that you can customize to build and deploy your code.
# Add steps that build, run tests, deploy, and more:
# https://aka.ms/yaml
trigger:
- master
pool:
vmImage: 'ubuntu-latest'
steps:
- script: echo Hello, world!
displayName: 'Run a one-line script'
- script: |
echo Add other tasks to build, test, and deploy your project.
echo See https://aka.ms/yaml
displayName: 'Run a multi-line script'
stages:
- stage: merge
displayName: 'merge'
jobs:
- template: ./templates/merge.yml
- stage: format
dependsOn: merge
displayName: 'format'
jobs:
- job: format
displayName: 'clang'
pool:
vmImage: ubuntu-latest
steps:
- template: ./templates/format-check.yml
- stage: build
displayName: 'build'
dependsOn: format
jobs:
- template: ./templates/build-standard.yml

18
.ci/yuzu-verify.yml Normal file
View File

@ -0,0 +1,18 @@
stages:
- stage: format
displayName: 'format'
jobs:
- job: format
displayName: 'clang'
pool:
vmImage: ubuntu-latest
steps:
- template: ./templates/format-check.yml
parameters:
artifactSource: 'false'
- stage: build
displayName: 'build'
dependsOn: format
jobs:
- template: ./templates/build-standard.yml
- template: ./templates/build-testing.yml

View File

@ -1,19 +0,0 @@
# Starter pipeline
# Start with a minimal pipeline that you can customize to build and deploy your code.
# Add steps that build, run tests, deploy, and more:
# https://aka.ms/yaml
trigger:
- master
pool:
vmImage: 'ubuntu-latest'
steps:
- script: echo Hello, world!
displayName: 'Run a one-line script'
- script: |
echo Add other tasks to build, test, and deploy your project.
echo See https://aka.ms/yaml
displayName: 'Run a multi-line script'