Skip to content

Commit

Permalink
Merge pull request #59949 from nyalldawson/grass7im
Browse files Browse the repository at this point in the history 
Fix grass imagery tests on newer GRASS versions
  • Loading branch information
@alexbruy
alexbruy authored Dec 18, 2024
2 parents c95d63c + c180972 commit 53e174f
Show file tree
Hide file tree
Showing 3 changed files with 206 additions and 40 deletions.
1 change: 0 additions & 1 deletion .ci/test_blocklist_qt6.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,5 @@ PyQgsServerWFS
ProcessingQgisAlgorithmsTestPt2
ProcessingQgisAlgorithmsTestPt4
ProcessingGdalAlgorithmsVectorTest
ProcessingGrassAlgorithmsImageryTest
# PyQgsProviderRegistry runs fine locally on Fedora:rawhide but not on CI
PyQgsProviderRegistry
132 changes: 130 additions & 2 deletions python/plugins/grassprovider/tests/AlgorithmsTestBase.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,15 @@
from numpy import nan_to_num
from copy import deepcopy

from qgis.PyQt.QtCore import QT_VERSION
from qgis.core import (
Qgis,
QgsVectorLayer,
QgsRasterLayer,
QgsCoordinateReferenceSystem,
QgsFeatureRequest,
QgsMapLayer,
QgsMeshLayer,
QgsProject,
QgsApplication,
QgsProcessingContext,
Expand All @@ -48,14 +51,23 @@
)
from qgis.analysis import QgsNativeAlgorithms
from qgis.testing import _UnexpectedSuccess, QgisTestCase, start_app

from utilities import unitTestDataPath
from grassprovider.grass_utils import GrassUtils

import processing

gdal.UseExceptions()


def GDAL_COMPUTE_VERSION(maj, min, rev):
return (maj) * 1000000 + (min) * 10000 + (rev) * 100


def GRASS_VERSION_INT(version: str) -> int:
parts = version.split(".")
return int(parts[0]) * 1000000 + int(parts[1]) * 10000 + int(parts[2]) * 100


def processingTestDataPath():
return os.path.join(os.path.dirname(__file__), "testdata")

Expand All @@ -73,6 +85,108 @@ def test_algorithms(self):

if "tests" in algorithm_tests and algorithm_tests["tests"] is not None:
for idx, algtest in enumerate(algorithm_tests["tests"]):
condition = algtest.get("condition")
if condition:
grass_condition = condition.get("grass")
if grass_condition:
grass_version = GRASS_VERSION_INT(GrassUtils.installedVersion())
less_than_condition = grass_condition.get("less_than")
if less_than_condition:
if grass_version >= less_than_condition:
print(
"!!! Skipping {}, requires GRASS < {}, have version {}".format(
algtest["name"],
less_than_condition,
grass_version,
)
)
continue
at_least_condition = grass_condition.get("at_least")
if at_least_condition:
if grass_version < at_least_condition:
print(
"!!! Skipping {}, requires GRASS >= {}, have version {}".format(
algtest["name"],
at_least_condition,
grass_version,
)
)
continue

geos_condition = condition.get("geos")
if geos_condition:
less_than_condition = geos_condition.get("less_than")
if less_than_condition:
if Qgis.geosVersionInt() >= less_than_condition:
print(
"!!! Skipping {}, requires GEOS < {}, have version {}".format(
algtest["name"],
less_than_condition,
Qgis.geosVersionInt(),
)
)
continue
at_least_condition = geos_condition.get("at_least")
if at_least_condition:
if Qgis.geosVersionInt() < at_least_condition:
print(
"!!! Skipping {}, requires GEOS >= {}, have version {}".format(
algtest["name"],
at_least_condition,
Qgis.geosVersionInt(),
)
)
continue
gdal_condition = condition.get("gdal")
if gdal_condition:
less_than_condition = gdal_condition.get("less_than")
if less_than_condition:
if (
int(gdal.VersionInfo("VERSION_NUM"))
>= less_than_condition
):
print(
"!!! Skipping {}, requires GDAL < {}, have version {}".format(
algtest["name"],
less_than_condition,
gdal.VersionInfo("VERSION_NUM"),
)
)
continue
at_least_condition = gdal_condition.get("at_least")
if at_least_condition:
if (
int(gdal.VersionInfo("VERSION_NUM"))
< at_least_condition
):
print(
"!!! Skipping {}, requires GDAL >= {}, have version {}".format(
algtest["name"],
at_least_condition,
gdal.VersionInfo("VERSION_NUM"),
)
)
continue
qt_condition = condition.get("qt")
if qt_condition:
less_than_condition = qt_condition.get("less_than")
if less_than_condition:
if QT_VERSION >= less_than_condition:
print(
"!!! Skipping {}, requires Qt < {}, have version {}".format(
algtest["name"], less_than_condition, QT_VERSION
)
)
continue
at_least_condition = qt_condition.get("at_least")
if at_least_condition:
if QT_VERSION < at_least_condition:
print(
"!!! Skipping {}, requires Qt >= {}, have version {}".format(
algtest["name"], at_least_condition, QT_VERSION
)
)
continue
print(
'About to start {} of {}: "{}"'.format(
idx, len(algorithm_tests["tests"]), algtest["name"]
Expand Down Expand Up @@ -136,6 +250,12 @@ def check_algorithm(self, name, defs):
# ignore user setting for invalid geometry handling
context = QgsProcessingContext()
context.setProject(QgsProject.instance())
if "ellipsoid" in defs:
# depending on the project settings, we can't always rely
# on QgsProject.ellipsoid() returning the same ellipsoid as was
# specified in the test definition. So just force ensure that the
# context's ellipsoid is the desired one
context.setEllipsoid(defs["ellipsoid"])

if "skipInvalid" in defs and defs["skipInvalid"]:
context.setInvalidGeometryCheck(
Expand Down Expand Up @@ -180,7 +300,7 @@ def load_param(self, param, id=None):
parameter based on its key `type` and return the appropriate parameter to pass to the algorithm.
"""
try:
if param["type"] in ("vector", "raster", "table"):
if param["type"] in ("vector", "raster", "table", "mesh"):
return self.load_layer(id, param).id()
elif param["type"] == "vrtlayers":
vals = []
Expand Down Expand Up @@ -290,6 +410,8 @@ def load_layer(self, id, param):
options = QgsRasterLayer.LayerOptions()
options.loadDefaultStyle = False
lyr = QgsRasterLayer(filepath, param["name"], "gdal", options)
elif param["type"] == "mesh":
lyr = QgsMeshLayer(filepath, param["name"], "mdal")

self.assertTrue(
lyr.isValid(), f'Could not load layer "{filepath}" from param {param}'
Expand Down Expand Up @@ -440,10 +562,16 @@ class GenericAlgorithmsTest(QgisTestCase):
@classmethod
def setUpClass(cls):
start_app()
from processing.core.Processing import Processing

Processing.initialize()
cls.cleanup_paths = []

@classmethod
def tearDownClass(cls):
from processing.core.Processing import Processing

Processing.deinitialize()
for path in cls.cleanup_paths:
shutil.rmtree(path)

Expand Down
113 changes: 76 additions & 37 deletions python/plugins/grassprovider/tests/testdata/grass_algorithms_imagery_tests.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -381,8 +381,13 @@ tests:
type: raster
results:
signaturefile:
type: file
name: expected/i.gensig.txt
type: regex
rules:
- '300\s*\n(?:\d+\s*\n)?7043\.06\s*\n1\.89772e\+07\s*'
- '1700\s*\n(?:\d+\s*\n)?7405\.93\s*\n2\.01921e\+07\s*'
- '6000\s*\n(?:\d+\s*\n)?7287\.21\s*\n1\.90221e\+07\s*'
- '7000\s*\n(?:\d+\s*\n)?7246\.81\s*\n1\.90226e\+07\s*'

- algorithm: grass7:i.gensigset
name: GRASS7 i.gensigset
Expand All @@ -399,8 +404,26 @@ tests:
type: raster
results:
signaturefile:
type: file
type: regex
name: expected/i.gensigset.txt
rules:
- 'subclass:\s*\n\s*pi:\s*0\.22802\s*\n\s*means:\s*9764\.5\s*\n\s*covar:\s*\n\s*3\.02234e\+06'
- 'subclass:\s*\n\s*pi:\s*0\.495197\s*\n\s*means:\s*5208\.5\s*\n\s*covar:\s*\n\s*7\.97435e\+06'
- 'subclass:\s*\n\s*pi:\s*0\.118668\s*\n\s*means:\s*997\.979\s*\n\s*covar:\s*\n\s*686411'
- 'subclass:\s*\n\s*pi:\s*0\.158115\s*\n\s*means:\s*13401\s*\n\s*covar:\s*\n\s*702077'
- 'subclass:\s*\n\s*pi:\s*0\.177696\s*\n\s*means:\s*13267\.4\s*\n\s*covar:\s*\n\s*1\.06971e\+06'
- 'subclass:\s*\n\s*pi:\s*0\.172485\s*\n\s*means:\s*3861\.81\s*\n\s*covar:\s*\n\s*2\.35556e\+06'
- 'subclass:\s*\n\s*pi:\s*0\.48383\s*\n\s*means:\s*8696\.48\s*\n\s*covar:\s*\n\s*7\.62681e\+06'
- 'subclass:\s*\n\s*pi:\s*0\.165989\s*\n\s*means:\s*1052\.05\s*\n\s*covar:\s*\n\s*692627'
- 'subclass:\s*\n\s*pi:\s*0\.135816\s*\n\s*means:\s*1158\.36\s*\n\s*covar:\s*\n\s*934515'
- 'subclass:\s*\n\s*pi:\s*0\.187835\s*\n\s*means:\s*11847\.9\s*\n\s*covar:\s*\n\s*1\.74438e\+06'
- 'subclass:\s*\n\s*pi:\s*0\.0829488\s*\n\s*means:\s*14066\.2\s*\n\s*covar:\s*\n\s*268579'
- 'subclass:\s*\n\s*pi:\s*0\.5934\s*\n\s*means:\s*6298\.74\s*\n\s*covar:\s*\n\s*8\.66516e\+06'
- 'subclass:\s*\n\s*pi:\s*0\.278456\s*\n\s*means:\s*3394\.59\s*\n\s*covar:\s*\n\s*2\.33337e\+06'
- 'subclass:\s*\n\s*pi:\s*0\.0849227\s*\n\s*means:\s*13998\.4\s*\n\s*covar:\s*\n\s*256746'
- 'subclass:\s*\n\s*pi:\s*0\.102818\s*\n\s*means:\s*686\.099\s*\n\s*covar:\s*\n\s*379342'
- 'subclass:\s*\n\s*pi:\s*0\.222279\s*\n\s*means:\s*11683\.6\s*\n\s*covar:\s*\n\s*1\.75208e\+06'
- 'subclass:\s*\n\s*pi:\s*0\.311523\s*\n\s*means:\s*7849\.26\s*\n\s*covar:\s*\n\s*3\.21781e\+06'

- algorithm: grass7:i.rgb.his
name: GRASS7 i.rgb.his
Expand All @@ -426,43 +449,53 @@ tests:
hash: 07578ad38cf948473a519f040acb0235f60a592904ac8ef46aa607e1
type: rasterhash

# Triggers a Python exception on the GRASS version used on the CI - disabling until fixed in GRASS
# - algorithm: grass7:i.pansharpen
# name: GRASS7 i.pansharpen
# params:
# GRASS_REGION_PARAMETER: '344500.0,358400.0,6682800.0,6693700.0'
# blue:
# name: raster_4class.tif
# type: raster
# green:
# name: raster_5class.tif
# type: raster
# method: 2
# pan:
# name: float_raster.tif
# type: raster
# red:
# name: raster_6class.tif
# type: raster
# results:
# blueoutput:
# hash:
# - 906de8be89e302057ed849d00eaf49332ecca73ffaba1374994f1a17
# - 6abf8904844b902dfb567246ad906ff51673fa61a821633eadc480c4
# type: rasterhash
# greenoutput:
# hash:
# - 588ad1ef8360ce903fc2defb1a1728a1dc8335d737d5fa77797605ed
# - 5c3ac77c7ac4a57b7a985541ce0c5f76c1709d8de7aa66b3acfa45d6
# type: rasterhash
# redoutput:
# hash:
# - ad80c0007faa1b0dc15c0b0c21ff4e0045ff5e67b454df0f65e68899
# - b488227a86c3ed58f5b2328238d109e69c55024f86efb1d4876de1f6
# type: rasterhash
#
- algorithm: grass7:i.pansharpen
name: GRASS7 i.pansharpen
condition:
grass:
# This test used to segfault on older GRASS releases
at_least: 8040000
params:
GRASS_REGION_PARAMETER: '344500.0,358400.0,6682800.0,6693700.0'
blue:
name: raster_4class.tif
type: raster
green:
name: raster_5class.tif
type: raster
method: 2
pan:
name: float_raster.tif
type: raster
red:
name: raster_6class.tif
type: raster
results:
blueoutput:
hash:
- 906de8be89e302057ed849d00eaf49332ecca73ffaba1374994f1a17
- 6abf8904844b902dfb567246ad906ff51673fa61a821633eadc480c4
type: rasterhash
greenoutput:
hash:
- 588ad1ef8360ce903fc2defb1a1728a1dc8335d737d5fa77797605ed
- 5c3ac77c7ac4a57b7a985541ce0c5f76c1709d8de7aa66b3acfa45d6
type: rasterhash
redoutput:
hash:
- ad80c0007faa1b0dc15c0b0c21ff4e0045ff5e67b454df0f65e68899
- b488227a86c3ed58f5b2328238d109e69c55024f86efb1d4876de1f6
type: rasterhash


- algorithm: grass7:i.smap
name: GRASS7 i.smap
condition:
grass:
# we can only run this test on older GRASS versions -- newer ones
# have the raster paths hardcoded into the signature file, and we can't handle
# that currently in the test suite
less_than: 8000000
params:
GRASS_REGION_PARAMETER: '344500.0,358400.0,6682800.0,6693700.0'
blocksize: 1024
Expand All @@ -484,6 +517,12 @@ tests:

- algorithm: grass7:i.maxlik
name: GRASS7 i.maxlik
condition:
grass:
# we can only run this test on older GRASS versions -- newer ones
# have the raster paths hardcoded into the signature file, and we can't handle
# that currently in the test suite
less_than: 8000000
params:
GRASS_REGION_PARAMETER: '344500.0,358400.0,6682800.0,6693700.0'
input:
Expand Down

0 comments on commit 53e174f

Please sign in to comment.