-
Notifications
You must be signed in to change notification settings - Fork 1.2k
/
azure-pypi-pipeline.yml
351 lines (338 loc) · 13 KB
/
azure-pypi-pipeline.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
name: PyPI_Packaging
# Trigger this build whenever the dev or release branches are updated,
# or on pull requests that affect PyPI-related files.
#
# Ideally we'd run this pipeline for all pull requests, but doing so consumes
# our limited number of slots on Azure and almost always just duplicates the
# build done in the main pipeline.
trigger:
branches:
include:
- dev
- release
pr:
branches:
include:
- dev
- release
paths:
include:
- azure-pypi-pipeline.yml
- build_scripts/pypi/*.py
- build_scripts/pypi/package_files/*
# Azure does not support an empty string as a default argument, so we use a
# special 'none' string for post_release_tag.
parameters:
- name: post_release_tag
displayName: Post Release Version Tag
type: string
default: 'none'
variables:
- name: post_release_tag_arg
${{ if eq(parameters.post_release_tag, 'none') }}:
value: ''
${{ else }}:
value: ${{ format('--post-release-tag {0}', parameters.post_release_tag) }}
stages:
- stage: Build_and_Package
jobs:
- job: Linux
strategy:
matrix:
Python38:
PYTHON_INTERPRETER: /opt/python/cp38-cp38/bin/python
PYTHON_TAG: cp38
Python39:
PYTHON_INTERPRETER: /opt/python/cp39-cp39/bin/python
PYTHON_TAG: cp39
Python310:
PYTHON_INTERPRETER: /opt/python/cp310-cp310/bin/python
PYTHON_TAG: cp310
Python311:
PYTHON_INTERPRETER: /opt/python/cp311-cp311/bin/python
PYTHON_TAG: cp311
Python312:
PYTHON_INTERPRETER: /opt/python/cp312-cp312/bin/python
PYTHON_TAG: cp312
timeoutInMinutes: 90
pool:
vmImage: Ubuntu-20.04
steps:
- bash: |
docker build -t manylinuxwithcmake build_scripts/pypi/docker
docker run --name usdmanylinux --rm -id -v $(Build.SourcesDirectory):/opt/USD -v /home/vsts/dist:/opt/USD-dist manylinuxwithcmake
displayName: 'Creating docker build environment'
- bash: |
# Terrible, terrible hack. The manylinux Docker image used to build the
# Python wheel does not include the corresponding Python shared library
# to link against. https://peps.python.org/pep-0513/#libpythonx-y-so-1
# describes why this is so. However, the FindPython CMake module used
# by USD's build system requires that the library exists and will error
# out otherwise, even though we explicitly avoid linking against Python
# via the PXR_PY_UNDEFINED_DYNAMIC_LOOKUP flag.
#
# To work around this, we create a dummy file for the library using
# the same logic as build_usd.py to determine where the library should
# exist (see GetPythonInfo). FindPython will see that the library exists
# and allow the build to continue. The file is 100% bogus, but the
# PXR_PY_UNDEFINED_DYNAMIC_LOOKUP flag will ensure that we never try to
# link against this library anyway, so it doesn't matter.
docker exec usdmanylinux $(PYTHON_INTERPRETER) -c "import pathlib,sysconfig; pathlib.Path(sysconfig.get_config_var('LIBDIR'), sysconfig.get_config_var('LDLIBRARY')).touch()"
docker exec usdmanylinux $(PYTHON_INTERPRETER) build_scripts/build_usd.py --build-args USD,"-DPXR_PY_UNDEFINED_DYNAMIC_LOOKUP=ON -DPXR_BUILD_USD_TOOLS=OFF -DPXR_INSTALL_LOCATION=../pxr/pluginfo" --no-materialx --no-imaging --no-examples --no-tutorials --build /opt/USD/gen/build --src /opt/USD/gen/src /opt/USD/inst -v
displayName: 'Building USD'
- bash: |
docker exec usdmanylinux mkdir ./packaging
docker exec usdmanylinux cp -R /opt/USD/inst ./packaging
docker exec usdmanylinux sh -c 'cp build_scripts/pypi/package_files/* ./packaging'
docker exec usdmanylinux sh -c 'cp LICENSE.txt ./packaging'
displayName: "Creating packaging directory"
- bash: |
docker exec -w /opt/USD/packaging usdmanylinux $(PYTHON_INTERPRETER) setup.py $(post_release_tag_arg) bdist_wheel --python-tag ${PYTHON_TAG}
displayName: 'Running setup.py'
- bash: |
docker exec usdmanylinux /bin/bash -c 'PYTHONPATH=/opt/USD/packaging/pypi/lib/python LD_LIBRARY_PATH=/opt/USD/packaging/pypi/lib:$LD_LIBRARY_PATH auditwheel repair packaging/dist/*.whl'
displayName: 'Running auditwheel repair (moves .so files into package)'
- bash: |
WHEEL_PACKAGE_NAME=`docker exec usdmanylinux ls wheelhouse`
docker exec usdmanylinux $(PYTHON_INTERPRETER) build_scripts/pypi/updatePluginfos.py "wheelhouse/$WHEEL_PACKAGE_NAME" "/opt/USD-dist/$WHEEL_PACKAGE_NAME"
displayName: 'Updating pluginfo paths'
- bash: |
docker stop usdmanylinux
displayName: 'Stopping docker container'
- task: PublishPipelineArtifact@0
inputs:
artifactName: dist-linux-$(PYTHON_TAG)
targetPath: /home/vsts/dist
- job: Windows
strategy:
matrix:
Python38:
PYTHON_VERSION_SPEC: 3.8
PYTHON_TAG: cp38
Python39:
PYTHON_VERSION_SPEC: 3.9
PYTHON_TAG: cp39
Python310:
PYTHON_VERSION_SPEC: 3.10
PYTHON_TAG: cp310
Python311:
PYTHON_VERSION_SPEC: 3.11
PYTHON_TAG: cp311
Python312:
PYTHON_VERSION_SPEC: 3.12
PYTHON_TAG: cp312
timeoutInMinutes: 90
pool:
vmImage: 'windows-2019'
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: $(PYTHON_VERSION_SPEC)
addToPath: true
- script: |
call C:\"Program Files (x86)"\"Microsoft Visual Studio"\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat
set BOOST_ROOT=
python --version
python build_scripts/build_usd.py --build-args USD,"-DPXR_ENABLE_PRECOMPILED_HEADERS=OFF -DPXR_PY_UNDEFINED_DYNAMIC_LOOKUP=ON -DPXR_BUILD_USD_TOOLS=OFF -DPXR_INSTALL_LOCATION=../pxr/pluginfo" --no-materialx --no-imaging --no-examples --no-tutorials --build %HOME%/USDgen/build --src %HOME%/USDgen/src %HOME%/USDinst -v
displayName: 'Building USD'
- script: |
dir
mkdir D:\packaging
xcopy /E /I %HOME%\USDinst D:\packaging\inst
copy build_scripts\pypi\package_files\* D:\packaging
copy LICENSE.txt D:\packaging
dir D:\packaging
dir D:\packaging\inst
displayName: "Creating packaging directory"
- script: |
python -m pip install wheel setuptools
displayName: 'Installing python packages'
- script: |
cd D:\packaging
dir
python setup.py $(post_release_tag_arg) bdist_wheel --python-tag $(PYTHON_TAG) --plat-name win_amd64
displayName: 'Running setup.py'
- task: PublishPipelineArtifact@0
inputs:
artifactName: dist-windows-$(PYTHON_TAG)
targetPath: D:\packaging\dist
- job: Mac
strategy:
matrix:
Python38:
PYTHON_VERSION_SPEC: 3.8
PYTHON_INTERPRETER: python3.8
PYTHON_TAG: cp38
Python39:
PYTHON_VERSION_SPEC: 3.9
PYTHON_INTERPRETER: python3.9
PYTHON_TAG: cp39
Python310:
PYTHON_VERSION_SPEC: 3.10
PYTHON_INTERPRETER: python3.10
PYTHON_TAG: cp310
Python311:
PYTHON_VERSION_SPEC: 3.11
PYTHON_INTERPRETER: python3.11
PYTHON_TAG: cp311
Python312:
PYTHON_VERSION_SPEC: 3.12
PYTHON_INTERPRETER: python3.12
PYTHON_TAG: cp312
timeoutInMinutes: 180
pool:
vmImage: 'macOS-13'
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: $(PYTHON_VERSION_SPEC)
addToPath: true
- script: |
sudo xcode-select -s /Applications/Xcode_14.1.app/Contents/Developer
$(PYTHON_INTERPRETER) build_scripts/build_usd.py --build-args USD,"-DPXR_PY_UNDEFINED_DYNAMIC_LOOKUP=ON -DPXR_BUILD_USD_TOOLS=OFF -DPXR_INSTALL_LOCATION=../pluginfo" --no-materialx --no-imaging --no-examples --no-tutorials --generator Xcode --build-target universal --build $HOME/USDgen/build --src $HOME/USDgen/src $HOME/USDinst -v
displayName: 'Building USD'
- bash: |
$(PYTHON_INTERPRETER) -m pip install delocate~=0.10.2 wheel setuptools
displayName: 'Installing python packages'
- bash: |
mkdir ./packaging
mkdir ./packaging/inst
cp -R $HOME/USDinst/* ./packaging/inst
cp build_scripts/pypi/package_files/* ./packaging
cp LICENSE.txt ./packaging
displayName: "Creating packaging directory"
- bash: |
cd ./packaging
$(PYTHON_INTERPRETER) setup.py $(post_release_tag_arg) bdist_wheel --python-tag ${PYTHON_TAG} --plat-name macosx_10_9_universal2
displayName: 'Running setup.py'
- bash: |
delocate-wheel -v -w dist-delocated packaging/dist/*
displayName: 'Running delocate (moves shared lib files into package)'
- bash: |
WHEEL_PACKAGE_NAME=`ls ./packaging/dist`
mkdir -p ./dist
$(PYTHON_INTERPRETER) build_scripts/pypi/updatePluginfos.py "./dist-delocated/$WHEEL_PACKAGE_NAME" "./dist/$WHEEL_PACKAGE_NAME"
displayName: 'Updating pluginfo paths'
- task: PublishPipelineArtifact@0
inputs:
artifactName: dist-mac-$(PYTHON_TAG)
targetPath: ./dist
- stage: Deliver
dependsOn: Build_and_Package
jobs:
- job: Collect_Packages
timeoutInMinutes: 15
pool:
vmImage: Ubuntu-20.04
steps:
- task: DownloadPipelineArtifact@2
displayName: 'Downloading all artifacts'
inputs:
source: current
# Not specifying artifact names so we get all of them
downloadPath: '$(Pipeline.Workspace)'
- bash: |
cd $(Pipeline.Workspace)
mkdir dist-final
cp dist*/*.whl dist-final
displayName: 'Collecting whl packages'
- task: PublishPipelineArtifact@0
displayName: 'Publishing in a single download'
inputs:
artifactName: dist
targetPath: '$(Pipeline.Workspace)/dist-final'
# The matrix below is verbose. There is a way to do this with a more n x m
# syntax using a foreach loop kind of construct. For now, opting for the below
# because it seems easier to read. An example of the other approach can be
# found here:
# https://github.com/nedrebo/parameterized-azure-jobs
- stage: Test_Packages
dependsOn: Deliver
jobs:
- job: Test_Install
strategy:
matrix:
Linux_Python38:
PYTHON_VERSION_SPEC: 3.8
IMAGE: 'Ubuntu-20.04'
PYTHON_INTERPRETER: python3
Linux_Python39:
PYTHON_VERSION_SPEC: 3.9
IMAGE: 'Ubuntu-20.04'
PYTHON_INTERPRETER: python3
Linux_Python310:
PYTHON_VERSION_SPEC: 3.10
IMAGE: 'Ubuntu-20.04'
PYTHON_INTERPRETER: python3
Linux_Python311:
PYTHON_VERSION_SPEC: 3.11
IMAGE: 'Ubuntu-20.04'
PYTHON_INTERPRETER: python3
Linux_Python312:
PYTHON_VERSION_SPEC: 3.12
IMAGE: 'Ubuntu-20.04'
PYTHON_INTERPRETER: python3
Windows_Python38:
PYTHON_VERSION_SPEC: 3.8
IMAGE: 'windows-2019'
PYTHON_INTERPRETER: python
Windows_Python39:
PYTHON_VERSION_SPEC: 3.9
IMAGE: 'windows-2019'
PYTHON_INTERPRETER: python
Windows_Python310:
PYTHON_VERSION_SPEC: 3.10
IMAGE: 'windows-2019'
PYTHON_INTERPRETER: python
Windows_Python311:
PYTHON_VERSION_SPEC: 3.11
IMAGE: 'windows-2019'
PYTHON_INTERPRETER: python
Windows_Python312:
PYTHON_VERSION_SPEC: 3.12
IMAGE: 'windows-2019'
PYTHON_INTERPRETER: python
Mac_Python38:
PYTHON_VERSION_SPEC: 3.8
IMAGE: 'macOS-13'
PYTHON_INTERPRETER: python3
Mac_Python39:
PYTHON_VERSION_SPEC: 3.9
IMAGE: 'macOS-13'
PYTHON_INTERPRETER: python3
Mac_Python310:
PYTHON_VERSION_SPEC: 3.10
IMAGE: 'macOS-13'
PYTHON_INTERPRETER: python3
Mac_Python311:
PYTHON_VERSION_SPEC: 3.11
IMAGE: 'macOS-13'
PYTHON_INTERPRETER: python3
Mac_Python312:
PYTHON_VERSION_SPEC: 3.12
IMAGE: 'macOS-13'
PYTHON_INTERPRETER: python3
timeoutInMinutes: 10
pool:
vmImage: '$(IMAGE)'
steps:
- task: UsePythonVersion@0
inputs:
versionSpec: $(PYTHON_VERSION_SPEC)
addToPath: true
- task: DownloadPipelineArtifact@2
displayName: 'Downloading dist package'
inputs:
source: current
artifact: dist
downloadPath: '$(Pipeline.Workspace)'
- script: |
$(PYTHON_INTERPRETER) --version
$(PYTHON_INTERPRETER) -m pip install pytest
$(PYTHON_INTERPRETER) -m pip install --no-index --find-links=file://$(Pipeline.Workspace) usd-core
py.test --junitxml TEST-usdinstall.xml build_scripts/pypi/test.py
displayName: "Testing fresh pip install"
- task: PublishTestResults@2
condition: succeededOrFailed()
inputs:
testRunTitle: 'Test results for $(IMAGE) Python $(PYTHON_VERSION_SPEC)'