Skip to content

Commit

Permalink
Merge pull request #9 from NFDI4BIOIMAGE/dev
Browse files Browse the repository at this point in the history
Tests and test data
  • Loading branch information
MicheleBortol authored Sep 24, 2024
2 parents 3dc3698 + 5ec15e1 commit f3345e1
Show file tree
Hide file tree
Showing 16 changed files with 55,842 additions and 61 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
build
.omero
node_modules
aurelia-webpack/aurelia_example/static
backbone-grunt/backbone_example/static
Expand Down
23 changes: 23 additions & 0 deletions .omeroci/app-build
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#! /usr/bin/env bash

source /infra/utils

set -e
set -u
set -x

cd $TARGET
DIR=$(setup_dir)

# TBD: unify on test or tests
# TBD: needs setup.py infra
# python setup.py test -t test* -i ${OMERO_DIST}/etc/ice.config -v

# Import test data
./.omeroci/test-data

set +u
export DJANGO_SETTINGS_MODULE=omeroweb.settings
export OMERODIR=/opt/omero/web/OMERO.web
export ICE_CONFIG=${OMERO_DIST}/etc/ice.config
pytest -vv
2 changes: 1 addition & 1 deletion .omeroci/app-config
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@ value=$(get_app_name)
# Configuration commands

su - omero-web -c "env OMERODIR=$OMERODIR /opt/omero/web/venv3/bin/omero config set omero.web.apps '[\"$value\"]'"
su - omero-web -c "env OMERODIR=$OMERODIR /opt/omero/web/venv3/bin/omero config set omero.web.omero_vitessce.serveraddress '"http://localhost:4080"'"
su - omero-web -c "env OMERODIR=$OMERODIR /opt/omero/web/venv3/bin/omero config set omero.web.omero_vitessce.serveraddress '\"http://localhost:4080\"'"
40 changes: 40 additions & 0 deletions .omeroci/test-data
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#! /usr/bin/env bash

export DATA_DIR=$TARGET"/test/data/MB266"

export PATH=$PATH:${OMERO_DIST}/bin

## Set up group and user for testing
omero login root@omero -w omero
omero group add TestGroup --type=read-annotate
omero user add test_user Test User --group-name=TestGroup --userpassword=password
omero logout

# Add test data
omero login test_user@omero -w password -g TestGroup
DATASET=$(omero obj new Dataset name=TestDataset) # dataset:1
PROJECT=$(omero obj new Project name=TestProject) # project:1
omero obj new ProjectDatasetLink parent=$PROJECT child=$DATASET

DAPI_IMAGE=$(omero import --output=ids "$DATA_DIR/MB266-DAPI.tiff" -T Dataset:id:1) # image:1
CELL_IMAGE=$(omero import --output=ids "$DATA_DIR/MB266-CELLS.png" -T Dataset:id:1) # image:2

CELL_FILE=$(omero upload "$DATA_DIR/cells.csv")
CELL_ATTACH=$(omero obj new FileAnnotation file=$CELL_FILE) # fileAnnotation:1
omero obj new DatasetAnnotationLink parent=$DATASET child=$CELL_ATTACH

EMBEDDING_FILE=$(omero upload "$DATA_DIR/embeddings.csv")
EMBEDDING_ATTACH=$(omero obj new FileAnnotation file=$EMBEDDING_FILE) # fileAnnotation:2
omero obj new DatasetAnnotationLink parent=$DATASET child=$EMBEDDING_ATTACH

TRANSCRIPT_FILE=$(omero upload "$DATA_DIR/transcripts.csv")
TRANSCRIPT_ATTACH=$(omero obj new FileAnnotation file=$TRANSCRIPT_FILE) # fileAnnotation:3
omero obj new DatasetAnnotationLink parent=$DATASET child=$TRANSCRIPT_ATTACH

EXPRESSION_FILE=$(omero upload "$DATA_DIR/feature_matrix.csv")
EXPRESSION_ATTACH=$(omero obj new FileAnnotation file=$EXPRESSION_FILE) # fileAnnotation:4
omero obj new DatasetAnnotationLink parent=$DATASET child=$EXPRESSION_ATTACH

CONFIG_FILE=$(omero upload "$DATA_DIR/VitessceConfig.json")
CONFIG_ATTACH=$(omero obj new FileAnnotation file=$CONFIG_FILE) # fileAnnotation:5
omero obj new DatasetAnnotationLink parent=$DATASET child=$CONFIG_ATTACH
29 changes: 28 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -136,13 +136,40 @@ Development

## Sources

The main sources are:
The main sources this project relies on are:
- omero-vitessce from Will Moore: https://github.com/will-moore/omero-vitessce
- cookiecutter-omero-webapp: https://github.com/ome/cookiecutter-omero-webapp
- `react_webapp` from omero-web-apps-examples: https://github.com/ome/omero-web-apps-examples/tree/master/react-webapp
- Vitessce python package used for generating config files http://python-docs.vitessce.io/
- omero-web-zarr to serve OME-NGFF images: https://github.com/ome/omero-web-zarr
- omero-openlink to serve images and other files: https://github.com/sukunis/OMERO.openlink
- omero-test-infra for testing: https://github.com/ome/omero-test-infra

## Tests

### Test dataset
The dataset used for automated testing is at: [test/data/MB266](test/data/MB266)
This is a cropped version derived from larger images and their associated data available at:
https://doi.org/10.5281/zenodo.13832665

The original data are available at:
https://www.ebi.ac.uk/biostudies/bioimages/studies/S-BIAD1093

The associated publication is available at:
https://www.biorxiv.org/content/10.1101/2024.04.03.586404v1

### omero-test-infra configuration
[omero-test-infra](https://github.com/ome/omero-test-infra) is used for automated tests with [github-actions](.github/workflows/omero_plugin.yml).
The workflow setups an omero installation in docker and then:
1) Runs [app-config](.omeroci/app-config) to set up the omero-web plugin configuration.
2) Runs [app-build](.omeroci/app-build) which:
- runs to create a test user/group and upload a [test dataset](test/data/MB266).
- runs pytest tests in: [test/integration](test/integration)

It is possible to perform these tests locally by:
1) Starting the docker daemon.
2) Cloning the omero-test infra repository inside the base folder of omero-vitessce: `git clone https://github.com/openmicroscopy/omero-test-infra .omero`
3) Running: `.omero/app-docker`

## React web app

Expand Down
5 changes: 3 additions & 2 deletions omero_vitessce/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def build_viewer_url(config_id):

def build_zarr_image_url(image_id):
""" Generates urls like:
http://localhost:4080/zarr/v0.4/image/99999.zarr/
http://localhost:4080/zarr/v0.4/image/99999.zarr
"""
return SERVER + "/zarr/v0.4/image/" + str(image_id) + ".zarr"

Expand Down Expand Up @@ -153,13 +153,14 @@ def create_config(config_args, obj_type, obj_id, conn):
# If an attachment/image is deleted/moved between the Vitessce right tab
# plugin is loaded and the form is submitted then it will not be found
# and will not be present in the cleaned_data -> None

file_names, file_urls, img_files, img_urls = get_files_images(
obj_type, obj_id, conn)
config_args = ConfigForm(data=config_args, file_names=file_names,
file_urls=file_urls, img_names=img_files,
img_urls=img_urls)
config_args.is_valid() # Generates the cleaned data
config_args = config_args.cleaned_data

description, name = get_details(obj_type, obj_id, conn)

vc = VitessceConfig(schema_version="1.0.16",
Expand Down
Binary file added test/data/MB266/MB266-CELLS.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added test/data/MB266/MB266-DAPI.tiff
Binary file not shown.
239 changes: 239 additions & 0 deletions test/data/MB266/VitessceConfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,239 @@
{
"version": "1.0.16",
"name": "TestDataset",
"description": "Generated with omero-vitessce",
"datasets": [
{
"uid": "A",
"name": "",
"files": [
{
"fileType": "obsSets.csv",
"url": "http://localhost:4080/webclient/annotation/1",
"options": {
"obsIndex": "cell_id",
"obsSets": [
{
"name": "Clustering",
"column": "label"
}
]
},
"coordinationValues": {
"obsType": "cell"
}
},
{
"fileType": "obsFeatureMatrix.csv",
"url": "http://localhost:4080/webclient/annotation/4"
},
{
"fileType": "obsEmbedding.csv",
"url": "http://localhost:4080/webclient/annotation/2",
"options": {
"obsIndex": "cell_id",
"obsEmbedding": [
"UMAP_1",
"UMAP_2"
]
},
"coordinationValues": {
"obsType": "cell",
"embeddingType": "cell"
}
},
{
"fileType": "obsLocations.csv",
"url": "http://localhost:4080/webclient/annotation/3",
"options": {
"obsIndex": "id",
"obsLocations": [
"x",
"y"
]
},
"coordinationValues": {
"obsType": "molecule"
}
},
{
"fileType": "obsLabels.csv",
"url": "http://localhost:4080/webclient/annotation/3",
"options": {
"obsIndex": "id",
"obsLabels": "gene"
},
"coordinationValues": {
"obsType": "molecule"
}
},
{
"fileType": "raster.json",
"options": {
"schemaVersion": "0.0.2",
"usePhysicalSizeScaling": true,
"images": [
{
"name": "Image 0",
"type": "ome-zarr",
"url": "http://localhost:4080/zarr/v0.4/image/1.zarr"
},
{
"name": "Segmentation",
"type": "ome-zarr",
"url": "http://localhost:4080/zarr/v0.4/image/2.zarr",
"metadata": {
"isBitmask": true
}
}
],
"renderLayers": [
"Image 0",
"Segmentation"
]
}
}
]
}
],
"coordinationSpace": {
"dataset": {
"A": "A"
},
"embeddingType": {
"A": "cell"
},
"spatialPointLayer": {
"A": {
"opacity": 1,
"radius": 2,
"visible": true
}
},
"spatialZoom": {
"A": 2
},
"spatialTargetX": {
"A": 0
},
"spatialTargetY": {
"A": 0
}
},
"layout": [
{
"component": "spatial",
"coordinationScopes": {
"dataset": "A",
"spatialPointLayer": "A"
},
"x": 0.0,
"y": 0.0,
"w": 4.0,
"h": 6.0
},
{
"component": "layerController",
"coordinationScopes": {
"dataset": "A",
"spatialPointLayer": "A"
},
"x": 3.0,
"y": 6.0,
"w": 1.0,
"h": 6.0
},
{
"component": "obsSets",
"coordinationScopes": {
"dataset": "A"
},
"x": 4.0,
"y": 6.0,
"w": 1.0,
"h": 6.0
},
{
"component": "featureList",
"coordinationScopes": {
"dataset": "A"
},
"x": 5.0,
"y": 6.0,
"w": 1.0,
"h": 6.0
},
{
"component": "scatterplot",
"coordinationScopes": {
"dataset": "A",
"embeddingType": "A"
},
"x": 4.0,
"y": 0.0,
"w": 4.0,
"h": 6.0
},
{
"component": "featureValueHistogram",
"coordinationScopes": {
"dataset": "A"
},
"x": 6.0,
"y": 6.0,
"w": 2.0,
"h": 6.0
},
{
"component": "obsSetSizes",
"coordinationScopes": {
"dataset": "A"
},
"x": 8.0,
"y": 6.0,
"w": 2.0,
"h": 6.0
},
{
"component": "obsSetFeatureValueDistribution",
"coordinationScopes": {
"dataset": "A"
},
"x": 10.0,
"y": 6.0,
"w": 2.0,
"h": 6.0
},
{
"component": "heatmap",
"coordinationScopes": {
"dataset": "A"
},
"x": 8.0,
"y": 0.0,
"w": 4.0,
"h": 6.0
},
{
"component": "description",
"coordinationScopes": {
"dataset": "A"
},
"x": 0.0,
"y": 6.0,
"w": 3.0,
"h": 3.0
},
{
"component": "status",
"coordinationScopes": {
"dataset": "A"
},
"x": 0.0,
"y": 9.0,
"w": 3.0,
"h": 3.0
}
],
"initStrategy": "auto"
}
Loading

0 comments on commit f3345e1

Please sign in to comment.