diff --git a/commons/src/main/java/org/open4goods/commons/dao/ProductRepository.java b/commons/src/main/java/org/open4goods/commons/dao/ProductRepository.java index e6ddaa03e..1e51144d1 100644 --- a/commons/src/main/java/org/open4goods/commons/dao/ProductRepository.java +++ b/commons/src/main/java/org/open4goods/commons/dao/ProductRepository.java @@ -56,9 +56,7 @@ public class ProductRepository { // The file queue implementation - // TODO(p3,conf) : Limit from conf - private BlockingQueue queue = new LinkedBlockingQueue<>(150000); - + private BlockingQueue queue; /** * !!!MAJOR CONST !!! Duration in ms where a price is considered to be valid. Only data with a @@ -87,6 +85,7 @@ public ProductRepository() { public ProductRepository(IndexationConfig indexationConfig) { logger.info("Starting file queue consumer thread, with bulk page size of {} items", indexationConfig.getProductsbulkPageSize() ); + this.queue = new LinkedBlockingQueue<>(indexationConfig.getProductsQueueMaxSize()); for (int i = 0; i < indexationConfig.getProductWorkers(); i++) { //TODO(p3,perf) : Virtual threads, but ko with visualVM profiling new Thread((new ProductIndexationWorker(this, indexationConfig.getProductsbulkPageSize(), indexationConfig.getPauseDuration(),"products-worker-"+i))).start(); diff --git a/commons/src/main/java/org/open4goods/commons/model/data/DataFragment.java b/commons/src/main/java/org/open4goods/commons/model/data/DataFragment.java index a78d74c65..ba6a96cf2 100644 --- a/commons/src/main/java/org/open4goods/commons/model/data/DataFragment.java +++ b/commons/src/main/java/org/open4goods/commons/model/data/DataFragment.java @@ -540,40 +540,40 @@ public void addReferentielAttribute(final String key, final String value) { * @return */ public void addAttribute(final String name, final String value, final String language, final Boolean ignoreCariageReturns,final Set multivalueSeparators ) { -// if (null == value || value.isBlank()) { -// logger.debug("Cannot add null or empty values for attribute " + name); -// return ; -// } -// -// // Evicting too big values -// // TODO(p3,conf) : From config -// if (value.length() > 10000) { -// logger.warn("Evicting a too big value for attribute {}:{}", name,value); -// return; -// } -// -// final Attribute attr = new Attribute(); -// -// // Attributye name normalisation -// String sanitizedName = IdHelper.normalizeAttributeName(name); -// // TODO(p2,conf) : From conf, could have the =, .. -// if (sanitizedName.endsWith(":")) { -// sanitizedName = sanitizedName.substring(0, sanitizedName.length() -1).trim(); -// } -// -// attr.setName(sanitizedName); -// -// if (ignoreCariageReturns.booleanValue()) { -// attr.setRawValue(value.trim().replaceAll("[\\r\\n]+", " ")); -// } else { -// attr.setRawValue(value); -// } -// -// attr.setLanguage(language); -// -// -// -// addAttribute(attr, multivalueSeparators); + if (null == value || value.isBlank()) { + logger.debug("Cannot add null or empty values for attribute " + name); + return ; + } + + // Evicting too big values + // TODO(p3,conf) : From config + if (value.length() > 10000) { + logger.warn("Evicting a too big value for attribute {}:{}", name,value); + return; + } + + final Attribute attr = new Attribute(); + + // Attributye name normalisation + String sanitizedName = IdHelper.normalizeAttributeName(name); + // TODO(p2,conf) : From conf, could have the =, .. + if (sanitizedName.endsWith(":")) { + sanitizedName = sanitizedName.substring(0, sanitizedName.length() -1).trim(); + } + + attr.setName(sanitizedName); + + if (ignoreCariageReturns.booleanValue()) { + attr.setRawValue(value.trim().replaceAll("[\\r\\n]+", " ")); + } else { + attr.setRawValue(value); + } + + attr.setLanguage(language); + + + + addAttribute(attr, multivalueSeparators); } /**