Queue tuning

open4good · Oct 7, 2024 · 5d9df8a · 5d9df8a
1 parent a2af1db
commit 5d9df8a
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 37 deletions.
diff --git a/commons/src/main/java/org/open4goods/commons/dao/ProductRepository.java b/commons/src/main/java/org/open4goods/commons/dao/ProductRepository.java
@@ -56,9 +56,7 @@ public class ProductRepository {
 
 
 	// The file queue implementation
-	// TODO(p3,conf) : Limit from conf
-	private BlockingQueue<Product> queue = new LinkedBlockingQueue<>(150000);
-
+	private BlockingQueue<Product> queue;
 
 	/**
 	 * !!!MAJOR CONST !!! Duration in ms where a price is considered to be valid. Only data with a
@@ -87,6 +85,7 @@ public ProductRepository() {
 	public ProductRepository(IndexationConfig indexationConfig) {
 		logger.info("Starting file queue consumer thread, with bulk page size of {} items", indexationConfig.getProductsbulkPageSize() );
 
+		this.queue = new LinkedBlockingQueue<>(indexationConfig.getProductsQueueMaxSize());
 		for (int i = 0; i < indexationConfig.getProductWorkers(); i++) {
 			//TODO(p3,perf) : Virtual threads, but ko with visualVM profiling
 			new Thread((new ProductIndexationWorker(this, indexationConfig.getProductsbulkPageSize(), indexationConfig.getPauseDuration(),"products-worker-"+i))).start();

diff --git a/commons/src/main/java/org/open4goods/commons/model/data/DataFragment.java b/commons/src/main/java/org/open4goods/commons/model/data/DataFragment.java
@@ -540,40 +540,40 @@ public void addReferentielAttribute(final String key, final String value) {
 	 * @return
 	 */
 	public void addAttribute(final String name, final String value, final String language, final Boolean ignoreCariageReturns,final Set<String> multivalueSeparators ) {
-//		if (null == value || value.isBlank()) {
-//			logger.debug("Cannot add null or empty values for attribute " + name);
-//			return ;
-//		}
-//
-//		// Evicting too big values
-//		// TODO(p3,conf) : From config
-//		if (value.length() > 10000) {
-//			logger.warn("Evicting a too big value for attribute {}:{}", name,value);
-//			return;
-//		}
-//		
-//		final Attribute attr = new Attribute();
-//
-//		// Attributye name normalisation
-//		String sanitizedName = IdHelper.normalizeAttributeName(name);
-//		// TODO(p2,conf) : From conf, could have the =, ..
-//		if (sanitizedName.endsWith(":")) {
-//			sanitizedName = sanitizedName.substring(0, sanitizedName.length() -1).trim();
-//		}
-//
-//		attr.setName(sanitizedName);
-//
-//		if (ignoreCariageReturns.booleanValue()) {
-//			attr.setRawValue(value.trim().replaceAll("[\\r\\n]+", " "));
-//		} else {
-//			attr.setRawValue(value);
-//		}
-//
-//		attr.setLanguage(language);
-//
-//
-//
-//		addAttribute(attr, multivalueSeparators);
+		if (null == value || value.isBlank()) {
+			logger.debug("Cannot add null or empty values for attribute " + name);
+			return ;
+		}
+
+		// Evicting too big values
+		// TODO(p3,conf) : From config
+		if (value.length() > 10000) {
+			logger.warn("Evicting a too big value for attribute {}:{}", name,value);
+			return;
+		}
+
+		final Attribute attr = new Attribute();
+
+		// Attributye name normalisation
+		String sanitizedName = IdHelper.normalizeAttributeName(name);
+		// TODO(p2,conf) : From conf, could have the =, ..
+		if (sanitizedName.endsWith(":")) {
+			sanitizedName = sanitizedName.substring(0, sanitizedName.length() -1).trim();
+		}
+
+		attr.setName(sanitizedName);
+
+		if (ignoreCariageReturns.booleanValue()) {
+			attr.setRawValue(value.trim().replaceAll("[\\r\\n]+", " "));
+		} else {
+			attr.setRawValue(value);
+		}
+
+		attr.setLanguage(language);
+
+
+
+		addAttribute(attr, multivalueSeparators);
 	}
 
 	/**