Verticals !

api/src/main/java/org/open4goods/api/controller/api/VerticalsGenerationController.java

@@ -95,15 +95,6 @@ public void exportMapping() throws ResourceNotFoundException, IOException {
 	}
 
 
-
-	@GetMapping(path="/mappings/generate/verticals")
-	@Operation(summary="Generate verticals from the mapping")
-	@PreAuthorize("hasAuthority('"+RolesConstants.ROLE_ADMIN+"')")
-	public List<VerticalConfig> generateVerticals() throws ResourceNotFoundException, IOException {
-		 return verticalsGenService.generateVerticals();
-
-	}	
-
 	@GetMapping(path="/assist/attributes/{vertical}")
 	@Operation(summary="Generate attributes coverage for a vertical")
 	@PreAuthorize("hasAuthority('"+RolesConstants.ROLE_ADMIN+"')")
@@ -118,8 +109,8 @@ public VerticalAttributesStats generateAttributesCoverage(@PathVariable String v
 	@Operation(summary="Generate the categories yaml fragment for a given match")
 	@PreAuthorize("hasAuthority('"+RolesConstants.ROLE_ADMIN+"')")
 	@Cacheable(keyGenerator = CacheConstants.KEY_GENERATOR, cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME)
-	public String generateCategoryMappingsFragment(@RequestParam String category) throws ResourceNotFoundException, IOException {
-		 return verticalsGenService.generateCategoryMappingFragmentFor(category);
+	public String generateCategoryMappingsFragment(@RequestParam String gtins) throws ResourceNotFoundException, IOException {
+		 return verticalsGenService.generateCategoryMappingFragmentForGtin(gtins.split(","));
 
 	}
 

api/src/main/java/org/open4goods/api/services/BatchService.java

@@ -102,8 +102,8 @@ public void batch(VerticalConfig vertical) {
 
 		logger.info("Loading products in memory for vertical {}", vertical);
 
-		// We take all products
-		allProducts = 	dataRepository.getProductsMatchingCategoriesOrVerticalId(vertical).collect(Collectors.toSet());
+		// We take all products that are typed with the given vertical
+		allProducts = 	dataRepository.getProductsMatchingVerticalId(vertical).collect(Collectors.toSet());
 
 		logger.info("Sanitisation of {} products", allProducts);
 

api/src/main/java/org/open4goods/api/services/VerticalsGenerationService.java

@@ -6,23 +6,27 @@
 import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.Map.Entry;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.NumberUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.open4goods.api.config.yml.VerticalsGenerationConfig;
 import org.open4goods.api.model.VerticalAttributesStats;
 import org.open4goods.api.model.VerticalCategoryMapping;
 import org.open4goods.commons.config.yml.ui.ProductI18nElements;
 import org.open4goods.commons.config.yml.ui.VerticalConfig;
 import org.open4goods.commons.dao.ProductRepository;
+import org.open4goods.commons.exceptions.ResourceNotFoundException;
 import org.open4goods.commons.helper.IdHelper;
 import org.open4goods.commons.model.product.Product;
 import org.open4goods.commons.services.EvaluationService;
@@ -302,18 +306,6 @@ public Map<String, VerticalCategoryMapping> getMappings() {
 		return sortedMappings;
 	}
 
-
-	public List<VerticalConfig> generateVerticals() {
-		List<VerticalConfig> ret = new ArrayList<VerticalConfig>();
-
-		sortedMappings.entrySet().stream()
-			.filter(e->e.getValue().getAssociatedCategories().size() > 3)
-			.forEach(cat -> {				
-				ret.add(generateVertical(cat));				
-			});
-		return ret;
-	}
-
 
 
 	/**
@@ -340,48 +332,6 @@ public VerticalAttributesStats attributesStats(String vertical) {
 		return ret;
 	}
 
-
-	/**
-	 * Generate a vertical stub, using our matching categories detected and adding informations through AI
-	 * @param cat
-	 * @return
-	 */
-	private VerticalConfig generateVertical(Entry<String, VerticalCategoryMapping> cat) {
-
-		VerticalConfig v = new VerticalConfig();
-		v.getMatchingCategories().add(cat.getKey());
-		v.getMatchingCategories().addAll(cat.getValue().getAssociatedCategories().keySet());
-
-		Map<String, String> datas = aiDatas(v);
-
-		Integer resolvedTaxonomy = resolveGoogleTaxonomy(datas.get("googleTaxonomy"));
-		if (null != resolvedTaxonomy) {
-			v.setGoogleTaxonomyId(resolvedTaxonomy);
-			LOGGER.warn("solved taxonomy for {} - {} ({})", cat.getKey(), datas,googleTaxonomyService.getTaxonomyName(resolvedTaxonomy) );
-
-			String englishName = datas.get("englishName");
-			String frenchName = datas.get("frenchName");
-
-			if (!StringUtils.isEmpty(englishName)) {
-				v.setId(IdHelper.brandName(englishName).toLowerCase());
-
-				ProductI18nElements fr = new ProductI18nElements();
-				fr.setVerticalHomeUrl(frenchName);
-				// TODO(p1, features) : Complete with other datas
-
-				v.getI18n().put("fr",fr );
-
-			}
-
-		} else {
-			LOGGER.warn("Unsolved taxonomy for {} - {}", cat.getKey(), datas);
-		}
-
-		return v;
-	}
-
-
-
 	/**
 	 * A prompt used to enrich the VerticalConfig
 	 * @param v
@@ -416,35 +366,38 @@ Strictly according to english google product taxonomy (https://www.google.com/ba
 
 
 	/**
-	 * Resolve a google taxonomy ID
-	 * @param string
-	 * @return
-	 */
-	private Integer resolveGoogleTaxonomy(String string) {
-		Integer ret = googleTaxonomyService.resolve(string);
-		LOGGER.info("Resolved google taxonomy for {} : {}",string,null); 
-		return ret;
-	}
-
-
-
-	/**
-	 * Generate the yaml fragment for a given category match
-	 * @param category
+	 * Generate the yaml categories mapping fragment from sample products
+	 * @param gtin
 	 * @return
 	 */
-	public String generateCategoryMappingFragmentFor(String category) {
-
-		VerticalCategoryMapping mapping = sortedMappings.get(category);
-
-		StringBuilder ret = new StringBuilder();
-		ret.append("matchingCategories:").append("\n");
-		ret.append("    - \"").append(category).append("\"\n");
-		if (null != mapping) {
-			for (String cat : mapping.getAssociatedCategories().keySet()) {
-				ret.append("    - \"").append(cat).append("\"\n");
+	public String generateCategoryMappingFragmentForGtin(String[] gtins) {
+
+		Map<String, Set<String>> matchingCategories = new HashMap<String, Set<String>>();
+		matchingCategories.put("all", new HashSet<String>());
+
+		for (String gtin : gtins) {
+			Product sample;
+			try {
+				if (NumberUtils.isDigits(gtin.trim())) {					
+					sample = repository.getById(Long.valueOf(gtin.trim()));
+					sample.getCategoriesByDatasources().entrySet().forEach(e -> {
+						if (!matchingCategories.containsKey(e.getKey())) {
+							matchingCategories.put(e.getKey(), new HashSet<String>());
+						}
+						matchingCategories.get(e.getKey()).add(e.getValue());
+
+					});
+				}
+			} catch (Exception e) {
+				LOGGER.warn("Cannot generate matching categories data : {}", e);
 			}
 		}
+
+
+		Map<String,Object> retMAp = new HashMap<String, Object>();
+		retMAp.put("matchingCategories", matchingCategories);
+		String ret = serialisationService.toYaml(retMAp);
+		ret = ret.replaceFirst("---", "");
 		return ret.toString();
 	}
 
@@ -467,7 +420,8 @@ public String verticalTemplate(String id, String googleTaxonomyId, String matchi
 
 			context.put("id",id );
 			context.put("googleTaxonomyId", googleTaxonomyId);
-			context.put("matchingCategories", generateCategoryMappingFragmentFor(matchingCategories));
+			// Here is a tweak, we provide some sample products coma separated
+			context.put("matchingCategories", generateCategoryMappingFragmentForGtin(matchingCategories.split(",")));
 			context.put("urlPrefix", urlPrefix);
 			context.put("h1Prefix", h1Prefix);
 			context.put("verticalHomeUrl", verticalHomeUrl);
@@ -482,26 +436,30 @@ public String verticalTemplate(String id, String googleTaxonomyId, String matchi
 
 	}
 
-	public void verticalTemplatetoFile(String googleTaxonomyId, String matchingCategories,String urlPrefix, String h1Prefix, String verticalHomeUrl, String verticalHomeTitle)  {
-
-		//TODO(p3, conf) : from conf 
-		String id = IdHelper.normalizeFileName(googleTaxonomyService.getLocalizedTaxonomy().get("en").get(Integer.valueOf(googleTaxonomyId)).getLast());
+	/**
+	 * Generate a vertical to a local file
+	 * 
+	 * @param googleTaxonomyId
+	 * @param matchingCategories
+	 * @param urlPrefix
+	 * @param h1Prefix
+	 * @param verticalHomeUrl
+	 * @param verticalHomeTitle
+	 */
+	public void verticalTemplatetoFile(String googleTaxonomyId, String matchingCategories, String urlPrefix, String h1Prefix, String verticalHomeUrl, String verticalHomeTitle) {
 
-		File f = new File("/opt/open4goods/tmp/");
-		f.mkdirs();
-		f = new File(f.getAbsolutePath() + "/" + id+".yml");
-
+		// TODO(p3, conf) : from conf
 		try {
-
+			String id = IdHelper.normalizeFileName(googleTaxonomyService.byId(Integer.valueOf(googleTaxonomyId)).getGoogleNames().i18n("en"));
+
+			File f = new File("/opt/open4goods/tmp/");
+			f.mkdirs();
+			f = new File(f.getAbsolutePath() + "/" + id + ".yml");
+
 			FileUtils.write(f, verticalTemplate(id, googleTaxonomyId, matchingCategories, urlPrefix, h1Prefix, verticalHomeUrl, verticalHomeTitle));
 		} catch (IOException e) {
-			LOGGER.error("Error while writing template file", e);
+			LOGGER.error("Error while writing template file for gtaxonomy {} ", googleTaxonomyId, e);
 		}
-
-
 	}
-
-
-
-
+
 }

api/src/main/java/org/open4goods/api/services/aggregation/services/realtime/TaxonomyRealTimeAggregationService.java

@@ -75,7 +75,7 @@ public void onProduct(Product data, VerticalConfig vConf) throws AggregationSkip
 		////////////////////////////
 		// Setting vertical from category
 		////////////////////////////
-		VerticalConfig vertical = verticalService.getVerticalForCategories(data.getDatasourceCategories());
+		VerticalConfig vertical = verticalService.getVerticalForCategories(data.getCategoriesByDatasources());
 		if (null != vertical) {
 			if ( null != data.getVertical() && !vertical.getId().equals(data.getVertical())) {
 				dedicatedLogger.warn("Will erase existing vertical {} with {} for product {}, because of category {}", data.getVertical(), vertical.getId(), data.bestName());

commons/src/main/java/org/open4goods/commons/config/yml/ui/VerticalConfig.java

@@ -91,14 +91,15 @@ public class VerticalConfig{
 
 
 	/**
-	 * The categories that MUST BE PRESENT to associate to this vertical
+	 * The categories that MUST BE PRESENT to associate to this vertical. Prefixed by datasource on which it applies, using "all" for all datasources
 	 */
-	private Set<String> matchingCategories = new HashSet<>();
 
+	private Map<String,Set<String>> matchingCategories = new HashMap<>();
+
 	/**
 	 * The categories that MUST NOT BE PRESENT to associate to this vertical
 	 */
-	private Set<String> unmatchingCategories = new HashSet<>();
+//	private Set<String> unmatchingCategories = new HashSet<>();
 
 	/**
 	 * The attributes that must be present. If not, the product will have excluded set to true
@@ -616,24 +617,21 @@ public void setScoringAggregationConfig(ScoringAggregationConfig scoringAggregat
 		this.scoringAggregationConfig = scoringAggregationConfig;
 	}
 
-
-
-
-	public Set<String> getMatchingCategories() {
+	public Map<String, Set<String>> getMatchingCategories() {
 		return matchingCategories;
 	}
 
-	public void setMatchingCategories(Set<String> matchingCategories) {
+	public void setMatchingCategories(Map<String, Set<String>> matchingCategories) {
 		this.matchingCategories = matchingCategories;
 	}
-
-	public Set<String> getUnmatchingCategories() {
-		return unmatchingCategories;
-	}
 
-	public void setUnmatchingCategories(Set<String> unmatchingCategories) {
-		this.unmatchingCategories = unmatchingCategories;
-	}
+//	public Set<String> getUnmatchingCategories() {
+//		return unmatchingCategories;
+//	}
+//
+//	public void setUnmatchingCategories(Set<String> unmatchingCategories) {
+//		this.unmatchingCategories = unmatchingCategories;
+//	}
 
 	public Map<String, Double> getEcoscoreConfig() {
 		return ecoscoreConfig;

commons/src/main/java/org/open4goods/commons/dao/ProductRepository.java

@@ -126,11 +126,7 @@ public ProductRepository(IndexationConfig indexationConfig) {
 	 */
 	public Stream<Product> getProductsMatchingCategoriesOrVerticalId(VerticalConfig v) {
 		Criteria c = new Criteria("datasourceCategories").in(v.getMatchingCategories())
-		// TODO : Add exclusion
-//				.and(new Criteria("datasourceCategories").notIn(v.getMatchingCategories()))
 				.or(new Criteria("vertical").is(v.getId()));
-// TODO : Add or get by taxonomyId
-
 
 		final NativeQuery initialQuery = new NativeQueryBuilder().withQuery(new CriteriaQuery(c)).build();
 
@@ -139,6 +135,16 @@ public Stream<Product> getProductsMatchingCategoriesOrVerticalId(VerticalConfig
 
 	}
 
+	public Stream<Product> getProductsMatchingVerticalId(VerticalConfig v) {
+		Criteria c = new Criteria("vertical").is(v.getId());
+
+		final NativeQuery initialQuery = new NativeQueryBuilder().withQuery(new CriteriaQuery(c)).build();
+
+		return elasticsearchOperations.searchForStream(initialQuery, Product.class, current_index).stream()
+				.map(SearchHit::getContent);
+
+	}
+
 	/**
 	 * Export all aggregated data
 	 * 

commons/src/main/java/org/open4goods/commons/services/SerialisationService.java

@@ -4,9 +4,7 @@
 import java.io.InputStream;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Set;
 
-import org.open4goods.commons.model.data.DataFragment;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
@@ -19,8 +17,10 @@
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.databind.type.CollectionType;
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator;
 
 @Service
 public class SerialisationService {
@@ -38,11 +38,13 @@ public class SerialisationService {
 
 	private final ObjectWriter jsonMapperWithPretttyPrint = new ObjectMapper().writerWithDefaultPrettyPrinter();
 
-	private final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory())
+	private final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()	.disable(YAMLGenerator.Feature.SPLIT_LINES))
 			.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
-
 			.setSerializationInclusion(Include.NON_EMPTY)
-			.setSerializationInclusion(Include.NON_NULL);
+			.setSerializationInclusion(Include.NON_NULL)
+			.enable(SerializationFeature.INDENT_OUTPUT)
+
+			;
 
 	public SerialisationService() {
 		super();