Expose arguments via config

planetiler-core/src/main/java/com/onthegomap/planetiler/Planetiler.java

@@ -95,8 +95,7 @@ public class Planetiler {
   private Planetiler(Arguments arguments) {
     this.arguments = arguments;
     stats = arguments.getStats();
-    overallTimer = stats.startStage("overall");
-    LogUtil.clearStage();
+    overallTimer = stats.startStageQuietly("overall");
     config = PlanetilerConfig.from(arguments);
     tmpDir = arguments.file("tmpdir", "temp directory", Path.of("data", "tmp"));
     onlyDownloadSources = arguments.getBoolean("only_download", "download source data then exit", false);
@@ -115,16 +114,6 @@ public static Planetiler create(Arguments arguments) {
     return new Planetiler(arguments);
   }
 
-  /**
-   * Returns a new empty runner that will get configuration from {@code arguments} to the main method, JVM properties,
-   * environmental variables, or a config file specified in {@code config} argument.
-   *
-   * @param arguments array of string arguments provided to {@code public static void main(String[] args)} entrypoint
-   */
-  public static Planetiler create(String... arguments) {
-    return new Planetiler(Arguments.fromArgsOrConfigFile(arguments));
-  }
-
   /**
    * Adds a new {@code .osm.pbf} source that will be processed when {@link #run()} is called.
    * <p>

planetiler-core/src/main/java/com/onthegomap/planetiler/config/Arguments.java

@@ -1,5 +1,7 @@
 package com.onthegomap.planetiler.config;
 
+import com.google.common.collect.HashMultiset;
+import com.google.common.collect.Multiset;
 import com.onthegomap.planetiler.geo.GeoUtils;
 import com.onthegomap.planetiler.stats.Stats;
 import java.io.IOException;
@@ -8,13 +10,15 @@
 import java.time.Duration;
 import java.time.format.DateTimeParseException;
 import java.time.temporal.ChronoUnit;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.TreeMap;
-import java.util.function.Function;
+import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
 import java.util.stream.Stream;
 import org.locationtech.jts.geom.Envelope;
@@ -29,11 +33,22 @@ public class Arguments {
 
   private static final Logger LOGGER = LoggerFactory.getLogger(Arguments.class);
 
-  private final Function<String, String> provider;
+  private final UnaryOperator<String> provider;
+  private final Supplier<? extends Collection<String>> keys;
   private boolean silent = false;
 
-  private Arguments(UnaryOperator<String> provider) {
+  private Arguments(UnaryOperator<String> provider, Supplier<? extends Collection<String>> keys) {
     this.provider = provider;
+    this.keys = keys;
+  }
+
+  private static Arguments from(UnaryOperator<String> provider, Supplier<? extends Collection<String>> rawKeys,
+    UnaryOperator<String> forward, UnaryOperator<String> reverse) {
+    Supplier<List<String>> keys = () -> rawKeys.get().stream().flatMap(key -> {
+      String reversed = reverse.apply(key);
+      return key.equalsIgnoreCase(reversed) ? Stream.empty() : Stream.of(reversed);
+    }).toList();
+    return new Arguments(key -> provider.apply(forward.apply(key)), keys);
   }
 
   /**
@@ -42,7 +57,17 @@ private Arguments(UnaryOperator<String> provider) {
    * For example to set {@code key=value}: {@code java -Dplanetiler.key=value -jar ...}
    */
   public static Arguments fromJvmProperties() {
-    return new Arguments(key -> System.getProperty("planetiler." + key));
+    return fromJvmProperties(
+      System::getProperty,
+      () -> System.getProperties().stringPropertyNames()
+    );
+  }
+
+  static Arguments fromJvmProperties(UnaryOperator<String> getter, Supplier<? extends Collection<String>> keys) {
+    return from(getter, keys,
+      key -> "planetiler." + key.toLowerCase(Locale.ROOT),
+      key -> key.replaceFirst("^planetiler\\.", "").toLowerCase(Locale.ROOT)
+    );
   }
 
   /**
@@ -51,7 +76,27 @@ public static Arguments fromJvmProperties() {
    * For example to set {@code key=value}: {@code PLANETILER_KEY=value java -jar ...}
    */
   public static Arguments fromEnvironment() {
-    return new Arguments(key -> System.getenv("PLANETILER_" + key.toUpperCase(Locale.ROOT)));
+    return fromEnvironment(
+      System::getenv,
+      () -> System.getenv().keySet()
+    );
+  }
+
+  static Arguments fromEnvironment(UnaryOperator<String> getter, Supplier<Set<String>> keys) {
+    return from(getter, keys,
+      key -> "PLANETILER_" + key.toUpperCase(Locale.ROOT),
+      key -> key.replaceFirst("^PLANETILER_", "").toLowerCase(Locale.ROOT)
+    );
+  }
+
+  /**
+   * Returns arguments parsed from a {@link Properties} object.
+   */
+  public static Arguments from(Properties properties) {
+    return new Arguments(
+      properties::getProperty,
+      properties::stringPropertyNames
+    );
   }
 
   /**
@@ -97,7 +142,7 @@ public static Arguments fromConfigFile(Path path) {
     Properties properties = new Properties();
     try (var reader = Files.newBufferedReader(path)) {
       properties.load(reader);
-      return new Arguments(properties::getProperty);
+      return from(properties);
     } catch (IOException e) {
       throw new IllegalArgumentException("Unable to load config file: " + path, e);
     }
@@ -147,7 +192,7 @@ public static Arguments fromEnvOrArgs(String... args) {
   }
 
   public static Arguments of(Map<String, String> map) {
-    return new Arguments(map::get);
+    return new Arguments(map::get, map::keySet);
   }
 
   /** Shorthand for {@link #of(Map)} which constructs the map from a list of key/value pairs. */
@@ -177,10 +222,20 @@ private String get(String key) {
    * @return arguments instance that checks {@code this} first and if a match is not found then {@code other}
    */
   public Arguments orElse(Arguments other) {
-    return new Arguments(key -> {
-      String ourResult = get(key);
-      return ourResult != null ? ourResult : other.get(key);
-    });
+    var result = new Arguments(
+      key -> {
+        String ourResult = get(key);
+        return ourResult != null ? ourResult : other.get(key);
+      },
+      () -> Stream.concat(
+        other.keys.get().stream(),
+        keys.get().stream()
+      ).distinct().toList()
+    );
+    if (silent) {
+      result.silence();
+    }
+    return result;
   }
 
   String getArg(String key) {
@@ -218,7 +273,7 @@ public Envelope bounds(String key, String description) {
     return result;
   }
 
-  private void logArgValue(String key, String description, Object result) {
+  protected void logArgValue(String key, String description, Object result) {
     if (!silent) {
       LOGGER.debug("argument: {}={} ({})", key, result, description);
     }
@@ -332,13 +387,13 @@ public int threads() {
   public Stats getStats() {
     String prometheus = getArg("pushgateway");
     if (prometheus != null && !prometheus.isBlank()) {
-      LOGGER.info("Using prometheus push gateway stats");
+      LOGGER.info("argument: stats=use prometheus push gateway stats");
       String job = getString("pushgateway.job", "prometheus pushgateway job ID", "planetiler");
       Duration interval = getDuration("pushgateway.interval", "how often to send stats to prometheus push gateway",
         "15s");
       return Stats.prometheusPushGateway(prometheus, job, interval);
     } else {
-      LOGGER.info("Using in-memory stats");
+      LOGGER.info("argument: stats=use in-memory stats");
       return Stats.inMemory();
     }
   }
@@ -390,4 +445,35 @@ public long getLong(String key, String description, long defaultValue) {
     logArgValue(key, description, parsed);
     return parsed;
   }
+
+  /**
+   * Returns a map from all the arguments provided to their values.
+   */
+  public Map<String, String> toMap() {
+    Map<String, String> result = new HashMap<>();
+    for (var key : keys.get()) {
+      result.put(key, get(key));
+    }
+    return result;
+  }
+
+  /** Returns a copy of this {@code Arguments} instance that logs each extracted argument value exactly once. */
+  public Arguments withExactlyOnceLogging() {
+    Multiset<String> logged = HashMultiset.create();
+    return new Arguments(this.provider, this.keys) {
+      @Override
+      protected void logArgValue(String key, String description, Object result) {
+        int count = logged.add(key, 1);
+        if (count == 0) {
+          super.logArgValue(key, description, result);
+        } else if (count == 3000) {
+          LOGGER.warn("Too many requests for argument '{}', result should be cached", key);
+        }
+      }
+    };
+  }
+
+  public boolean silenced() {
+    return silent;
+  }
 }

planetiler-core/src/main/java/com/onthegomap/planetiler/config/PlanetilerConfig.java

@@ -124,7 +124,7 @@ public static PlanetilerConfig from(Arguments arguments) {
       maxzoom,
       renderMaxzoom,
       arguments.getBoolean("skip_mbtiles_index_creation", "skip adding index to mbtiles file", false),
-      arguments.getBoolean("optimize_db", "optimize mbtiles after writing", false),
+      arguments.getBoolean("optimize_db", "Vacuum analyze mbtiles after writing", false),
       arguments.getBoolean("emit_tiles_in_order", "emit tiles in index order", true),
       arguments.getBoolean("force", "overwriting output file and ignore disk/RAM warnings", false),
       arguments.getBoolean("gzip_temp", "gzip temporary feature storage (uses more CPU, but less disk space)", false),

planetiler-core/src/main/java/com/onthegomap/planetiler/expression/DataType.java

@@ -32,13 +32,21 @@ public enum DataType implements BiFunction<WithTags, String, Object> {
     this(id, (d, k) -> parser.apply(d.getTag(k)), parser);
   }
 
-  @Override
-  public Object apply(WithTags withTags, String string) {
-    return this.getter.apply(withTags, string);
-  }
-
-  public Object convertFrom(Object value) {
-    return this.parser.apply(value);
+  /** Returns the data type associated with {@code value}, or {@link #GET_TAG} as a fallback. */
+  public static DataType typeOf(Object value) {
+    if (value instanceof String) {
+      return GET_STRING;
+    } else if (value instanceof Integer) {
+      return GET_INT;
+    } else if (value instanceof Long) {
+      return GET_LONG;
+    } else if (value instanceof Double) {
+      return GET_DOUBLE;
+    } else if (value instanceof Boolean) {
+      return GET_BOOLEAN;
+    } else {
+      return GET_TAG;
+    }
   }
 
   /** Returns the data type associated with {@code id}, or {@link #GET_TAG} as a fallback. */
@@ -51,6 +59,15 @@ public static DataType from(String id) {
     return GET_TAG;
   }
 
+  @Override
+  public Object apply(WithTags withTags, String string) {
+    return this.getter.apply(withTags, string);
+  }
+
+  public Object convertFrom(Object value) {
+    return this.parser.apply(value);
+  }
+
   public String id() {
     return id;
   }

planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Stats.java

@@ -62,11 +62,31 @@ default void printSummary() {
    * Also sets the "stage" prefix that shows up in the logs to {@code name}.
    */
   default Timers.Finishable startStage(String name) {
-    LogUtil.setStage(name);
-    var timer = timers().startTimer(name);
+    return startStage(name, true);
+  }
+
+  /**
+   * Same as {@link #startStage(String)} except does not log that it started, or set the logging prefix.
+   */
+  default Timers.Finishable startStageQuietly(String name) {
+    return startStage(name, false);
+  }
+
+  /**
+   * Records that a long-running task with {@code name} has started and returns a handle to call when finished.
+   * <p>
+   * Also sets the "stage" prefix that shows up in the logs to {@code name} if {@code log} is true.
+   */
+  default Timers.Finishable startStage(String name, boolean log) {
+    if (log) {
+      LogUtil.setStage(name);
+    }
+    var timer = timers().startTimer(name, log);
     return () -> {
       timer.stop();
-      LogUtil.clearStage();
+      if (log) {
+        LogUtil.clearStage();
+      }
     };
   }
 

planetiler-core/src/main/java/com/onthegomap/planetiler/stats/Timers.java

@@ -91,16 +91,22 @@ private List<String> getStageDetails(String name, boolean pad) {
   }
 
   public Finishable startTimer(String name) {
+    return startTimer(name, true);
+  }
+
+  public Finishable startTimer(String name, boolean logStart) {
     Timer timer = Timer.start();
     Stage stage = new Stage(timer);
     timers.put(name, stage);
     Stage last = currentStage.getAndSet(stage);
-    LOGGER.info("");
-    LOGGER.info("Starting...");
+    if (logStart) {
+      LOGGER.info("");
+      LOGGER.info("Starting...");
+    }
     return () -> {
-      LOGGER.info("Finished in " + timers.get(name).timer.stop());
+      LOGGER.info("Finished in {}", timers.get(name).timer.stop());
       for (var details : getStageDetails(name, true)) {
-        LOGGER.info("  " + details);
+        LOGGER.info("  {}", details);
       }
       currentStage.set(last);
     };

planetiler-core/src/main/java/com/onthegomap/planetiler/util/FileWatcher.java

@@ -1,7 +1,5 @@
 package com.onthegomap.planetiler.util;
 
-import static com.onthegomap.planetiler.util.Exceptions.throwFatalException;
-
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -104,19 +102,4 @@ public void pollForChanges(Duration delay, FunctionThatThrows<Set<Path>, Set<Pat
       }
     }
   }
-
-  @FunctionalInterface
-  public interface FunctionThatThrows<I, O> {
-
-    @SuppressWarnings("java:S112")
-    O apply(I value) throws Exception;
-
-    default O runAndWrapException(I value) {
-      try {
-        return apply(value);
-      } catch (Exception e) {
-        return throwFatalException(e);
-      }
-    }
-  }
 }

planetiler-core/src/main/java/com/onthegomap/planetiler/util/FunctionThatThrows.java

@@ -0,0 +1,18 @@
+package com.onthegomap.planetiler.util;
+
+import static com.onthegomap.planetiler.util.Exceptions.throwFatalException;
+
+@FunctionalInterface
+public interface FunctionThatThrows<I, O> {
+
+  @SuppressWarnings("java:S112")
+  O apply(I value) throws Exception;
+
+  default O runAndWrapException(I value) {
+    try {
+      return apply(value);
+    } catch (Exception e) {
+      return throwFatalException(e);
+    }
+  }
+}