-
Notifications
You must be signed in to change notification settings - Fork 47
274 lines (223 loc) · 8.67 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
name: ci
on:
push:
# A branch github-ci-updates can be created and used for ci
# experiments and tweaks.
branches: [ "develop", "master", "github-ci-updates" ]
pull_request:
branches: [ "develop", "master" ]
permissions:
contents: read
jobs:
# Lint, format check, and all tests.
base-ci:
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
matrix:
python_version: [ '3.8', '3.9', '3.10', '3.11' ]
steps:
- name: Install Dependencies
run: |
sudo apt-get update -y
sudo apt-get install -y mecab mecab-ipadic-utf8
# Helper checks to find the mecab library path,
# so it can be exported before running tests.
# Without the export, natto-py fails on github.
# echo FIND THE LIB:
# which mecab
# actual=`readlink -f /usr/bin/mecab`
# ldd $actual
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python_version }}
cache: 'pip' # caching pip dependencies
- run: pip install -r requirements.txt
- name: Setup config
run: |
mkdir ${{ github.workspace }}/data
echo "ENV: dev" > ${{ github.workspace }}/lute/config/config.yml
echo "DATAPATH: ${{ github.workspace }}/data" >> ${{ github.workspace }}/lute/config/config.yml
echo "DBNAME: test_lute.db" >> ${{ github.workspace }}/lute/config/config.yml
ls ${{ github.workspace }}
cat ${{ github.workspace }}/lute/config/config.yml
# Playwright tests were hanging far too often on github ci,
# but not consistently. No idea what was going wrong.
# TODO github ci: fix playwright hanging.
## - name: Playwright install
## run: playwright install
## - name: Playwright smoke test
## run: |
## set -x
## inv playwright
- name: Test
run: |
# Have to explicitly set MECAB_PATH for natto-py.
export MECAB_PATH=/lib/x86_64-linux-gnu/libmecab.so.2
pytest
# Note that these seem to *pass* even if an acceptance
# test fails in GitHub actions ... which is annoying,
# because everything passes/fails correctly locally.
- name: Acceptance testing
run: |
export MECAB_PATH=/lib/x86_64-linux-gnu/libmecab.so.2
inv accept --headless -s
# Code style etc checks after functionality verified.
- name: Lint
run: inv lint
- name: Coding style
run: |
black --check .
- name: Remove mecab for smoke test without mecab
run: |
sudo apt-get remove -y mecab mecab-ipadic-utf8
# Run one particular sanity check.
#
# Note this isn't _completely_ valid because the test loads
# only supported language stories, whereas a prod release
# comes with _all_ stories pre-loaded and the invalid ones
# are deleted ...
- name: Smoke test no mecab
run: inv accept --headless -s -k test_unsupported_language_not_shown
- name: Check flit package
run: |
mkdir ${{ github.workspace }}/../lute_flit
cd ${{ github.workspace }}/../lute_flit
flit -f ${{ github.workspace }}/pyproject.toml install
python -m lute.main & # Start in background process
sleep 2
# Verify with utils script back in the workspace
# (verify.py is not included in flit install)
pushd ${{ github.workspace }}
python -m utils.verify 5000
popd
pkill -f "python -m lute.main" # Kill that process.
# Test build docker container and try running.
# Slightly wasteful re-setup of node.
docker-build:
runs-on: ubuntu-latest
timeout-minutes: 30
needs: base-ci
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: 3.11
cache: 'pip' # caching pip dependencies
- run: pip install -r requirements.txt
- name: Test docker build
run: |
docker build -f docker/Dockerfile --build-arg INSTALL_MECAB=false -t lute3 .
# Run container in the background, and check.
docker run -d -p 5000:5000 -v ./my_data:/lute_data -v ./my_backups:/lute_backup --name my-lute lute3:latest
sleep 10 # Give it a moment to start.
python -m utils.verify 5000
docker stop my-lute
docker rm my-lute
# Generate a coverage badge, don't worry if it fails.
# Uses https://github.com/Schneegans/dynamic-badges-action to update a secret gist
# (ID a15001ec2ff889f7be0b553df9881566) and an API token, per notes at
# https://nedbatchelder.com/blog/202209/making_a_coverage_badge.html.
# Slightly wasteful in that it re-runs the tests, but it's fastest.
coverage:
needs: base-ci
continue-on-error: true
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Install Dependencies
run: |
sudo apt-get update -y
sudo apt-get install -y mecab mecab-ipadic-utf8
# Helper checks to find the mecab library path,
# so it can be exported before running tests.
# Without the export, natto-py fails on github.
# echo FIND THE LIB:
# which mecab
# actual=`readlink -f /usr/bin/mecab`
# ldd $actual
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: '3.8'
cache: 'pip' # caching pip dependencies
- run: pip install -r requirements.txt
- name: Setup config
run: |
mkdir ${{ github.workspace }}/data
echo "ENV: dev" > ${{ github.workspace }}/lute/config/config.yml
echo "DATAPATH: ${{ github.workspace }}/data" >> ${{ github.workspace }}/lute/config/config.yml
echo "DBNAME: test_lute.db" >> ${{ github.workspace }}/lute/config/config.yml
ls ${{ github.workspace }}
cat ${{ github.workspace }}/lute/config/config.yml
- name: Generate coverage
run: |
# Have to explicitly set MECAB_PATH for natto-py.
export MECAB_PATH=/lib/x86_64-linux-gnu/libmecab.so.2
coverage run -m pytest tests/
coverage json --omit="tests/*"
export TOTAL=$(python -c "import json;print(json.load(open('coverage.json'))['totals']['percent_covered_display'])")
echo "total=$TOTAL" >> $GITHUB_ENV
echo "### Total coverage: ${TOTAL}%" >> $GITHUB_STEP_SUMMARY
- name: "Make badge"
uses: schneegans/[email protected]
with:
# GIST_TOKEN is a GitHub personal access token with scope "gist".
auth: ${{ secrets.GIST_TOKEN }}
gistID: a15001ec2ff889f7be0b553df9881566
filename: covbadge.json
label: coverage
message: ${{ env.total }}%
valColorRange: ${{ env.total }}
minColorRange: 50
maxColorRange: 80
# Ensure that basic things work correctly on Windows,
# particularly PlatformDirs package.
windows-ci:
runs-on: windows-latest
timeout-minutes: 30
strategy:
matrix:
python_version: [ '3.8', '3.9', '3.10', '3.11' ]
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python_version }}
cache: 'pip' # caching pip dependencies
- name: Setup config
run: |
mkdir ${{ github.workspace }}\data
echo ENV: dev > ${{ github.workspace }}\lute\config\config.yml
echo DATAPATH: ${{ github.workspace }}\data >> ${{ github.workspace }}\lute\config\config.yml
echo DBNAME: test_lute.db >> ${{ github.workspace }}\lute\config\config.yml
shell: cmd
- name: Replace config slashes.
run: |
$content = [System.IO.File]::ReadAllText("${{ github.workspace }}\lute\config\config.yml").Replace('\','/')
[System.IO.File]::WriteAllText("${{ github.workspace }}\lute\config\config.yml", $content)
Get-Content -Path "${{ github.workspace }}\lute\config\config.yml"
- run: pip install -r requirements.txt
# Run one particular sanity check.
- name: Smoke test no mecab
run: inv accept --headless -s -k test_unsupported_language_not_shown
- name: Remove config to force using prod config
run: del ${{ github.workspace }}\lute\config\config.yml
shell: cmd
- name: Install prod-like flit package
run: |
mkdir ${{ github.workspace }}\..\lute_flit
cd ${{ github.workspace }}\..\lute_flit
flit -f ${{ github.workspace }}\pyproject.toml install
shell: cmd
- name: Run Lute application
run: |
echo "python -m lute.main" > run-server.bat
cmd /c "START /b run-server.bat"
- name: Wait for application to start
run: Start-Sleep -Seconds 10
- name: Verify flit install
run: |
cd ${{ github.workspace }}
python -m utils.verify 5000