Skip to content

Update devstart to use new demo service. #1096

Update devstart to use new demo service.

Update devstart to use new demo service. #1096

Workflow file for this run

name: ci
on:
push:
# A branch github-ci can be created and used for ci
# experiments and tweaks.
branches: [ "develop", "master", "github-ci", "windows" ]
pull_request:
branches: [ "develop", "master" ]
permissions:
contents: read
jobs:
# Run all tests.
base-ci:
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
matrix:
python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ]
steps:
- name: Install Dependencies
run: |
sudo apt-get update -y
sudo apt-get install -y mecab mecab-ipadic-utf8
# Helper checks to find the mecab library path,
# so it can be exported before running tests.
# Without the export, natto-py fails on github.
# echo FIND THE LIB:
# which mecab
# actual=`readlink -f /usr/bin/mecab`
# ldd $actual
- uses: actions/checkout@v4
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python_version }}
cache: 'pip' # caching pip dependencies
- name: Install packages using flit
run: |
pip install flit
# --only-deps: Installs only the dependencies, not Lute itself
# --deps develop: both regular and optional dependencies.
flit install --only-deps --deps develop
- name: Setup config
run: |
mkdir ${{ github.workspace }}/data
echo "ENV: dev" > ${{ github.workspace }}/lute/config/config.yml
echo "DATAPATH: ${{ github.workspace }}/data" >> ${{ github.workspace }}/lute/config/config.yml
echo "DBNAME: test_lute.db" >> ${{ github.workspace }}/lute/config/config.yml
ls ${{ github.workspace }}
cat ${{ github.workspace }}/lute/config/config.yml
- name: Test
run: |
set -e
# Have to explicitly set MECAB_PATH for natto-py.
export MECAB_PATH=/lib/x86_64-linux-gnu/libmecab.so.2
pytest
# Skipping acceptance tests for 3.8, just to save some time.
- name: Acceptance testing
if: matrix.python_version != '3.8'
uses: nick-fields/retry@v3
with:
max_attempts: 3
timeout_minutes: 30
command: |
# handle errors manually, so that the retry works.
exit_code=0
# Have to explicitly set MECAB_PATH for natto-py.
export MECAB_PATH=/lib/x86_64-linux-gnu/libmecab.so.2
inv accept --show --verbose || exit_code=$?
if [ "$exit_code" -ne 0 ]; then
echo "Command failed with exit code $exit_code, retrying..."
fi
exit $exit_code
# Playwright tests were hanging far too often on github ci,
# but not consistently. No idea what was going wrong.
# TODO github ci: fix playwright hanging.
- name: Playwright install
run: playwright install
- name: Playwright smoke test
run: |
set -x
set -e
inv playwright
- name: Check flit package
run: |
mkdir ${{ github.workspace }}/../lute_flit
cd ${{ github.workspace }}/../lute_flit
flit -f ${{ github.workspace }}/pyproject.toml install
python -m lute.main & # Start in background process
sleep 2
# Verify with utils script back in the workspace
# (verify.py is not included in flit install)
pushd ${{ github.workspace }}
python -m utils.verify 5001
popd
pkill -f "python -m lute.main" # Kill that process.
# Lute should still work if the user doesn't have mecab installed.
no-mecab-check:
runs-on: ubuntu-latest
timeout-minutes: 10
strategy:
matrix:
# Only checking early and late versions.
python_version: [ '3.8', '3.11' ]
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python_version }}
cache: 'pip' # caching pip dependencies
- name: Install packages using flit
run: |
pip install flit
flit install --only-deps --deps develop
- name: Setup config
run: |
mkdir ${{ github.workspace }}/data
echo "ENV: dev" > ${{ github.workspace }}/lute/config/config.yml
echo "DATAPATH: ${{ github.workspace }}/data" >> ${{ github.workspace }}/lute/config/config.yml
echo "DBNAME: test_lute.db" >> ${{ github.workspace }}/lute/config/config.yml
ls ${{ github.workspace }}
cat ${{ github.workspace }}/lute/config/config.yml
# Run one particular sanity check.
#
# Note this isn't _completely_ valid because the test loads
# only supported language stories, whereas a prod release
# comes with _all_ stories pre-loaded and the invalid ones
# are deleted ...
- name: Smoke test no mecab
run: inv accept -s -k disabled_data_is_hidden
# Run all plugin tests.
#
# For each plugin:
# - install Lute requirements
# - install plugin reqs
# - run tests.
#
# The Lute requirements are installed first b/c the plugins may come
# with their own conflicting requirements. Doing a full req install
# will (hopefully) uncover conflicts.
plugins:
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
matrix:
python_version: [ '3.8', '3.9', '3.10', '3.11' ]
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python_version }}
cache: 'pip' # caching pip dependencies
# Plugins likely won't need this config file, but just in case ...
- name: Setup config
run: |
mkdir ${{ github.workspace }}/data
echo "ENV: dev" > ${{ github.workspace }}/lute/config/config.yml
echo "DATAPATH: ${{ github.workspace }}/data" >> ${{ github.workspace }}/lute/config/config.yml
echo "DBNAME: test_lute.db" >> ${{ github.workspace }}/lute/config/config.yml
ls ${{ github.workspace }}
cat ${{ github.workspace }}/lute/config/config.yml
- name: test all plugins
run: |
# Lute reqs, such as pytest.
# This also installs Lute itself,
# so that it can be found
# by each plugin's own "pip install ."
pip install flit
flit install --deps develop
for plugin in $(ls plugins); do
pushd plugins/$plugin
pip install .
# Note for future: some plugins may have extra reqs not covered by pip
# (e.g. mecab uses apt-get and exports etc). Idea for future: plugin
# could have a .github folder as well with additional setup scripts.
pytest tests
# pip uninstall $plugin -y
# NOTE: Not bothering to do an uninstall!
# if multiple plugins have different/clashing version requirements,
# perhaps it is best to run into problems in ci.
# This may ultimately come back to haunt me, but it will do for now.
popd
done
code-quality:
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
matrix:
python_version: [ '3.11' ]
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python_version }}
cache: 'pip' # caching pip dependencies
- run: |
pip install flit
flit install --only-deps --deps develop
- name: Lint
run: inv lint
- name: Coding style
run: |
black --check .
# Test build docker container and try running.
# Slightly wasteful re-setup of node.
docker-build:
runs-on: ubuntu-latest
timeout-minutes: 30
needs: base-ci
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: 3.11
cache: 'pip' # caching pip dependencies
- run: |
pip install flit
flit install --only-deps --deps develop
- name: Test docker build
run: |
docker build -f docker/Dockerfile --build-arg INSTALL_EVERYTHING=false -t lute3 .
# Run container in the background, and check.
docker run -d -p 5001:5001 -v ./my_data:/lute_data -v ./my_backups:/lute_backup --name my-lute lute3:latest
sleep 10 # Give it a moment to start.
python -m utils.verify 5001
docker stop my-lute
docker rm my-lute
# Ensure that basic things work correctly on Windows,
# particularly PlatformDirs package.
windows-ci:
runs-on: windows-latest
timeout-minutes: 30
strategy:
matrix:
python_version: [ '3.8', '3.9', '3.10', '3.11' ]
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python_version }}
cache: 'pip' # caching pip dependencies
- name: Setup config
run: |
mkdir ${{ github.workspace }}\data
echo ENV: dev > ${{ github.workspace }}\lute\config\config.yml
echo DATAPATH: ${{ github.workspace }}\data >> ${{ github.workspace }}\lute\config\config.yml
echo DBNAME: test_lute.db >> ${{ github.workspace }}\lute\config\config.yml
shell: cmd
- name: Replace config slashes.
run: |
$content = [System.IO.File]::ReadAllText("${{ github.workspace }}\lute\config\config.yml").Replace('\','/')
[System.IO.File]::WriteAllText("${{ github.workspace }}\lute\config\config.yml", $content)
Get-Content -Path "${{ github.workspace }}\lute\config\config.yml"
- run: |
pip install flit
flit install --only-deps --deps develop
# Can't get playwright to run reliably on Windows.
# - name: Playwright install
# run: playwright install
# - name: Playwright smoke test
# run: inv playwright || exit /b
# Now having problems with tests not working on windows ...
# getting failure message:
# javascript error: clear_datatable_state is not defined
#
# The above message is called from lute_test_client to clear book
# datatables state. This _used_ to work (e.g. in v3.3.0), and
# I can't track it down at the moment!!!!!!
#
# TODO ci: RESTORE AT LEAST ONE SANITY CHECK TEST ON WINDOWS.
#
# Run specific sanity check.
# Old tests no longer run -- datatables may have timing issues on Windows tests,
# tests were far too flaky.
# inv accept -s -k test_unsupported_language_not_shown || exit /b
# inv accept -s -k import_a_valid_term_file || exit /b
# - name: Smoke tests
# run: |
# inv accept -s -k test_updating_term_status_updates_the_reading_frame || exit /b
- name: Remove config to force using prod config
run: del ${{ github.workspace }}\lute\config\config.yml
shell: cmd
- name: Install prod-like flit package
run: |
mkdir ${{ github.workspace }}\..\lute_flit
cd ${{ github.workspace }}\..\lute_flit
flit -f ${{ github.workspace }}\pyproject.toml install
shell: cmd
- name: Run Lute application
run: |
echo "python -m lute.main" > run-server.bat
cmd /c "START /b run-server.bat"
- name: Wait for application to start
run: Start-Sleep -Seconds 10
- name: Verify flit install
run: |
cd ${{ github.workspace }}
python -m utils.verify 5001