AutoMapping: Fixed automatic output regions for object output

With this change, output object layers are taken into account when
determining the rule output regions.

The support for outputting objects by AutoMapping is still lacking
support for many cases, like polygon/polyline objects or the object
alignment setting, but it is at least meant to work for basic rectangle
or tile objects.

This change also adjusts tileRegionOfObjectGroup to the change that made
object position and size be stored in pixels rather than tiles
(55e7767), as was already done for
objectsToErase before (521a40a).

Closes #3473

NEWS.md

@@ -12,6 +12,7 @@
 * AutoMapping: When input regions are defined, match in order by default (#3559)
 * AutoMapping: Skip locked layers when applying rules (#3544)
 * AutoMapping: Fixed NoOverlappingOutput in case of multiple output indices (#3551)
+* AutoMapping: Fixed automatic output regions for object output (#3473)
 * Scripting: Added Object.setColorProperty and Object.setFloatProperty (#3423)
 * Scripting: Added tiled.projectFilePath
 * Scripting: Added tiled.versionLessThan

src/tiled/automapper.cpp

@@ -148,10 +148,9 @@ AutoMappingContext::AutoMappingContext(MapDocument *mapDocument)
 
 AutoMapper::AutoMapper(std::unique_ptr<Map> rulesMap, const QRegularExpression &mapNameFilter)
     : mRulesMap(std::move(rulesMap))
+    , mRulesMapRenderer(MapRenderer::create(mRulesMap.get()))
     , mMapNameFilter(mapNameFilter)
 {
-    Q_ASSERT(mRulesMap);
-
     setupRuleMapProperties();
 
     if (setupRuleMapLayers())
@@ -341,7 +340,6 @@ bool AutoMapper::setupRuleMapLayers()
     Q_ASSERT(setup.mInputLayerNames.isEmpty());
 
     QString error;
-    const auto renderer = MapRenderer::create(mRulesMap.get());
 
     for (Layer *layer : mRulesMap->allLayers()) {
         if (layer->isGroupLayer() || layer->isImageLayer())
@@ -403,8 +401,7 @@ bool AutoMapper::setupRuleMapLayers()
                     }
 
                     RuleOptionsArea &optionsArea = setup.mRuleOptionsAreas.emplace_back();
-                    optionsArea.area = QRectF(renderer->pixelToTileCoords(mapObject->bounds().topLeft()),
-                                              renderer->pixelToTileCoords(mapObject->bounds().bottomRight())).toAlignedRect();
+                    optionsArea.area = objectTileRect(*mRulesMapRenderer, *mapObject);
                     setupRuleOptionsArea(optionsArea, mapObject);
                 }
             } else {
@@ -575,8 +572,14 @@ void AutoMapper::setupRules()
             std::for_each(outputSet.layers.keyBegin(),
                           outputSet.layers.keyEnd(),
                           [&] (const Layer *layer) {
-                if (layer->isTileLayer())
-                    regionOutput |= static_cast<const TileLayer*>(layer)->region();
+                if (layer->isTileLayer()) {
+                    auto tileLayer = static_cast<const TileLayer*>(layer);
+                    regionOutput |= tileLayer->region();
+                } else if (layer->isObjectGroup()) {
+                    auto objectGroup = static_cast<const ObjectGroup*>(layer);
+                    regionOutput |= tileRegionOfObjectGroup(*mRulesMapRenderer,
+                                                            objectGroup);
+                }
             });
         }
     }
@@ -1169,7 +1172,8 @@ void AutoMapper::applyRule(const Rule &rule, QPoint pos,
                 targetLayer = context.outputTileLayers.value(targetName);
                 break;
             case Layer::ObjectGroupType:
-                outputLayerRegion = tileRegionOfObjectGroup(static_cast<const ObjectGroup*>(layer));
+                outputLayerRegion = tileRegionOfObjectGroup(*mRulesMapRenderer,
+                                                            static_cast<const ObjectGroup*>(layer));
                 targetLayer = context.outputObjectGroups.value(targetName);
                 break;
             case Layer::ImageLayerType:

src/tiled/automapper.h

@@ -44,6 +44,7 @@ namespace Tiled {
 class Layer;
 class Map;
 class MapObject;
+class MapRenderer;
 class ObjectGroup;
 class TileLayer;
 
@@ -419,6 +420,7 @@ class TILED_EDITOR_EXPORT AutoMapper : public QObject
      * Map containing the rules.
      */
     const std::unique_ptr<Map> mRulesMap;
+    const std::unique_ptr<MapRenderer> mRulesMapRenderer;
     const QRegularExpression mMapNameFilter;
 
     RuleMapSetup mRuleMapSetup;

src/tiled/automappingutils.cpp

@@ -29,66 +29,64 @@
 
 namespace Tiled {
 
+QRect objectTileRect(const MapRenderer &renderer,
+                     const MapObject &object)
+{
+    // TODO: we are checking bounds, which is only correct for rectangles and
+    // tile objects. polygons and polylines are not covered correctly by this
+    // erase method (we are in fact deleting too many objects)
+    // TODO2: toAlignedRect may even break rects.
+
+    // Convert the boundary of the object into tile space
+    const QRectF bounds = object.boundsUseTile();
+    const QPointF topLeft = renderer.pixelToTileCoords(bounds.topLeft());
+    const QPointF bottomRight = renderer.pixelToTileCoords(bounds.bottomRight());
+
+    return QRectF(topLeft, bottomRight).toAlignedRect();
+}
+
 QList<MapObject*> objectsToErase(const MapDocument *mapDocument,
                                  const ObjectGroup *layer,
                                  const QRegion &where)
 {
     QList<MapObject*> objectsToErase;
 
-    for (MapObject *obj : layer->objects()) {
-        // TODO: we are checking bounds, which is only correct for rectangles and
-        // tile objects. polygons and polylines are not covered correctly by this
-        // erase method (we are in fact deleting too many objects)
-        // TODO2: toAlignedRect may even break rects.
-
-        // Convert the boundary of the object into tile space
-        const QRectF objBounds = obj->boundsUseTile();
-        const QPointF tl = mapDocument->renderer()->pixelToTileCoords(objBounds.topLeft());
-        const QPointF tr = mapDocument->renderer()->pixelToTileCoords(objBounds.topRight());
-        const QPointF br = mapDocument->renderer()->pixelToTileCoords(objBounds.bottomRight());
-        const QPointF bl = mapDocument->renderer()->pixelToTileCoords(objBounds.bottomLeft());
-
-        QRectF objInTileSpace;
-        objInTileSpace.setTopLeft(tl);
-        objInTileSpace.setTopRight(tr);
-        objInTileSpace.setBottomRight(br);
-        objInTileSpace.setBottomLeft(bl);
-
-        const QRect objAlignedRect = objInTileSpace.toAlignedRect();
-        if (where.intersects(objAlignedRect))
-            objectsToErase.append(obj);
+    for (MapObject *object : layer->objects()) {
+        const QRect tileRect = objectTileRect(*mapDocument->renderer(), *object);
+        if (where.intersects(tileRect))
+            objectsToErase.append(object);
     }
 
     return objectsToErase;
 }
 
-QRegion tileRegionOfObjectGroup(const ObjectGroup *layer)
+QRegion tileRegionOfObjectGroup(const MapRenderer &renderer,
+                                const ObjectGroup *objectGroup)
 {
-    QRegion ret;
-    for (MapObject *obj : layer->objects()) {
-        // TODO: we are using bounds, which is only correct for rectangles and
-        // tile objects. polygons and polylines are not probably covering less
-        // tiles.
-        ret += obj->bounds().toAlignedRect();
-    }
-    return ret;
+    QRegion region;
+    for (const MapObject *object : objectGroup->objects())
+        region |= objectTileRect(renderer, *object);
+    return region;
 }
 
-QList<MapObject*> objectsInRegion(const ObjectGroup *layer, const QRegion &where)
+/**
+ * Returns the list of objects occupying the given rectangle (in pixels).
+ */
+QList<MapObject*> objectsInRegion(const ObjectGroup *objectGroup,
+                                  const QRectF &where)
 {
     QList<MapObject*> ret;
-    for (MapObject *obj : layer->objects()) {
+    for (MapObject *object : objectGroup->objects()) {
         // TODO: we are checking bounds, which is only correct for rectangles and
         // tile objects. polygons and polylines are not covered correctly by this
         // erase method (we are in fact deleting too many objects)
-        // TODO2: toAlignedRect may even break rects.
-        const QRect rect = obj->boundsUseTile().toAlignedRect();
+        const QRectF rect = object->boundsUseTile();
 
         // QRegion::intersects() returns false for empty regions even if they are
         // contained within the region, so we also check for containment of the
         // top left to include the case of zero size objects.
         if (where.intersects(rect) || where.contains(rect.topLeft()))
-            ret += obj;
+            ret += object;
     }
     return ret;
 }

src/tiled/automappingutils.h

@@ -26,17 +26,22 @@
 namespace Tiled {
 
 class MapObject;
+class MapRenderer;
 class ObjectGroup;
 
 class MapDocument;
 
+QRect objectTileRect(const MapRenderer &renderer,
+                     const MapObject &object);
+
 QList<MapObject*> objectsToErase(const MapDocument *mapDocument,
                                  const ObjectGroup *layer,
                                  const QRegion &where);
 
-QRegion tileRegionOfObjectGroup(const ObjectGroup *layer);
+QRegion tileRegionOfObjectGroup(const MapRenderer &renderer,
+                                const ObjectGroup *objectGroup);
 
-QList<MapObject*> objectsInRegion(const ObjectGroup *layer,
-                                  const QRegion &where);
+QList<MapObject*> objectsInRegion(const ObjectGroup *objectGroup,
+                                  const QRectF &where);
 
 } // namespace Tiled

tests/automapping/test_automapping.cpp

@@ -100,7 +100,7 @@ void test_AutoMapping::autoMap()
             const QPoint pos = it.key();
             const Cell &expected = it.value();
             const Cell &seen = mapTileLayer->cellAt(pos);
-            QCOMPARE(expected, seen);
+            QCOMPARE(seen, expected);
         }
     }
 }