Verticals generation

open4good · Dec 8, 2024 · f65f741 · f65f741
1 parent 686c357
commit f65f741
Show file tree

Hide file tree

Showing 7 changed files with 203 additions and 3,090 deletions.
diff --git a/api/src/main/java/org/open4goods/api/controller/api/VerticalsGenerationController.java b/api/src/main/java/org/open4goods/api/controller/api/VerticalsGenerationController.java
@@ -179,10 +179,12 @@ public void updateVerticalWithMappings(
 	@Operation(summary="Update the suggested attributes for a given vertical directly in the file !")
 	@PreAuthorize("hasAuthority('"+RolesConstants.ROLE_ADMIN+"')")
 	public void updateVerticalWithAttributes( 
-			@RequestParam	(defaultValue = "tv")									 String vertical) throws ResourceNotFoundException, IOException {
+			@RequestParam	(defaultValue = "tv")									 String vertical,
+			@RequestParam	(defaultValue = "10")									 Integer minCoverage
+			) throws ResourceNotFoundException, IOException {
 
 		//TODO(p2,conf) : from conf
- 		  verticalsGenService.updateVerticalFileWithAttributes("/home/goulven/git/open4goods/verticals/src/main/resources/verticals/"+vertical+".yml");
+ 		  verticalsGenService.updateVerticalFileWithAttributes("/home/goulven/git/open4goods/verticals/src/main/resources/verticals/"+vertical+".yml", minCoverage);
 
 	}
 

diff --git a/api/src/main/java/org/open4goods/api/services/VerticalsGenerationService.java b/api/src/main/java/org/open4goods/api/services/VerticalsGenerationService.java
@@ -318,7 +318,7 @@ public VerticalAttributesStats attributesStats(String vertical) {
 		VerticalAttributesStats ret = new VerticalAttributesStats() ;
 		if (null != vc) {
 			LOGGER.info("Attributes stats for vertical {} is running",vertical);
-			repository.exportVerticalWithValidDate(vc, false).forEach(p -> {
+			repository.exportVerticalWithValidDate(vc, true).forEach(p -> {
 				ret.process(p.getAttributes().getAll());
 			});
 
@@ -531,8 +531,9 @@ public void updateVerticalFileWithCategories(Integer minOffers, String fileName)
 	 * Update a vertical file with predicted attributes  in the vertical yaml files
 	 * @param minOffers
 	 * @param fileName
+	 * @param minCoverage 
 	 */
-	public void updateVerticalFileWithAttributes(String fileName) {
+	public void updateVerticalFileWithAttributes(String fileName, int minCoverage) {
 		File file = new File(fileName);
 		try {
 			VerticalConfig vc = verticalConfigservice.getConfigById(file.getName().substring(0, file.getName().length()-4));
@@ -544,7 +545,7 @@ public void updateVerticalFileWithAttributes(String fileName) {
 
 			String newContent = originalContent.substring(0, startIndex);
 			newContent += "  configs:\n";
-			newContent += generateAttributesMapping(vc);
+			newContent += generateAttributesMapping(vc, minCoverage);
 			newContent += originalContent.substring(endIndex);
 
 			FileUtils.writeStringToFile(file, newContent, Charset.defaultCharset());
@@ -557,10 +558,11 @@ public void updateVerticalFileWithAttributes(String fileName) {
 
 	/**
 	 * Generate the advised attributes for a vertical
+	 * @param minCoverage 
 	 * @param vertical
 	 * @return
 	 */
-	public String generateAttributesMapping(VerticalConfig verticalConfig) {
+	public String generateAttributesMapping(VerticalConfig verticalConfig, int minCoverage) {
 		LOGGER.info("Generating attributes mapping for {}", verticalConfig);
 		VerticalAttributesStats stats = attributesStats(verticalConfig.getId());
 
@@ -587,8 +589,8 @@ public String generateAttributesMapping(VerticalConfig verticalConfig) {
 
 			if (!exclusions.contains(cat.getKey())) {
 
-				// TODO(p2,conf) : limit from conf
-				if (Double.valueOf(cat.getValue().getHits() / Double.valueOf(totalItems) * 100.0).intValue() > 15) {
+
+				if (Double.valueOf(cat.getValue().getHits() / Double.valueOf(totalItems) * 100.0).intValue() > minCoverage) {
 					LOGGER.info("Generating template for attribute : {}", cat.getKey());
 					// TODO(conf,p2) : numberofsamples from conf
 					ret += attributeConfigTemplate(cat, totalItems,10);					

diff --git a/api/src/main/resources/templates/attribute-definition.yml b/api/src/main/resources/templates/attribute-definition.yml
@@ -4,7 +4,7 @@
   # Generated on [(${date})], coverage was [(${coveragePercent})]% ([(${attrHits})]/[(${totalHits})]) . 
   # TOP attributes values are :
 [(${attributesSamples})]   
-  ################################################################################
+  ##################################
   - key: "[(${name})]"
     filteringType: "[(${type})]"
     asScore: false

diff --git a/api/src/main/resources/templates/vertical-definition.yml b/api/src/main/resources/templates/vertical-definition.yml
@@ -17,6 +17,11 @@ googleTaxonomyId: [(${googleTaxonomyId})]
 # Indicates if the vertical must be UI / sitemap / search rendered
 enabled: false
 
+# GenAiConfig for this vertical
+genAiConfig:
+  # If false, will bypass the generativ ia texts generation  
+  enabled: true
+
 #####################################################################################################################################
 # I18N CONFIGURATION
 # Configure in a i18n way all the texts of a product, including the Url, the title, the description, the open graph metas, ....
@@ -52,7 +57,7 @@ i18n:
       # Text displaid as title for the vertical
       # TODO : Make verticalHomeTitle/ verticalHomeDescription templatable
       verticalHomeTitle: "[(${verticalHomeTitle})]"
-      # Text displaid as title for the vertical
+      # Text displaid as jumbotron for the vertical
       # verticalHomeDescription: ""  
 
      # A mapping beetween custom page names behind this vertical and wiki pages
@@ -63,12 +68,7 @@ i18n:
 #          title: "OLED, QLED,LCD, ... Quelle technologie choisir ?"
 #          faIcon: "fa-star"
 
-
-# GenAiConfig for this vertical
-genAiConfig:
-  # If false, will bypass the generativ ia texts generation  
-  enabled: false
-
+
 
 ##############################################################################
 # Eco score configuration 

diff --git a/verticals/src/main/resources/verticals/_default.yml b/verticals/src/main/resources/verticals/_default.yml
@@ -24,8 +24,16 @@ generationExcludedFromAttributesMatching:
   - AVAILABILITY
   - BRAND_NAME
   - BRAND
+  - MARQUE
   - NOMFOURNISSEUR
   - THEME
+  - PRODUCT_NAME
+  - CATEGORIE
+  - DESCRIPTIF
+  - TITRE,
+  - PRICE
+  - SIZE_STOCK_STATUS
+  - PRICE_NOREBATE  
 
 # If product categories contains one of this token, this will be unmatched
 excludingTokensFromCategoriesMatching:
@@ -276,6 +284,7 @@ attributesConfig:
     - "REFURBISHED"
     - "ETAT_PRODUIT"
     - "DISPONIBILITE"
+    - "SIZE_STOCK_STATUS"
 
     # Noisy items   
     - "ADVERT_RAKUPON_AMOUNT"
@@ -310,7 +319,9 @@ attributesConfig:
     - "STOCK_STATUS"
     - "WEB_OFFER"
     - "BASE_PRICE"
+    - "PRICE_NOREBATE"
     - "PRIX"
+    - "PRICE"
     - "REVIEWS"
     - "SAVING"
     - "SAVINGS_PERCENT"
@@ -354,6 +365,7 @@ attributesConfig:
       - "CLASSE ECO"
       - "CLASSE ENERGÉTIQUE"
       - "CLASSE D'EFFICACITÉ ÉNERGÉTIQUE"
+      - "CLASSE D'EFFICACITE ENERGETIQUE"
       - "CLASSE D'EFFICACITÉ ÉNERGÉTIQUE (HDR)"
       - "ENERGY EFFICIENCY CLASS"
       - "CLASSE_ENERGY"
@@ -477,7 +489,10 @@ attributesConfig:
       all:
       - "ANNÉE DE LANCEMENT"
       - "ANNÉE DE SORTIE"
+      - "ANNEE DE SORTIE"                
       - "ANNÉE DE MODÈLE"
+      - "ANNEE DE LANCEMENT"
+      - "YEAR"
 
     parser:
       normalize: true