diff --git a/api/src/main/java/org/open4goods/api/controller/api/BatchController.java b/api/src/main/java/org/open4goods/api/controller/api/BatchController.java index e9117d7f0..b2e061f6c 100644 --- a/api/src/main/java/org/open4goods/api/controller/api/BatchController.java +++ b/api/src/main/java/org/open4goods/api/controller/api/BatchController.java @@ -84,6 +84,7 @@ public void batchVertical( @PathVariable @NotBlank final String vertical ) throw @PostMapping(path="/score/{vertical}") @Operation(summary="Score a specific vertical") + @PreAuthorize("hasAuthority('"+RolesConstants.ROLE_ADMIN+"')") public void scoreFromName( @PathVariable @NotBlank final String vertical ) throws InvalidParameterException, JsonParseException, JsonMappingException, IOException, InterruptedException{ aggregationFacadeService. score(verticalConfigService.getConfigById(vertical)); @@ -92,25 +93,37 @@ public void scoreFromName( @PathVariable @NotBlank final String vertical ) thro @GetMapping("/score/") @Operation(summary="Score all verticals (sanitisation + launch the scheduled batch that score all verticals)") + @PreAuthorize("hasAuthority('"+RolesConstants.ROLE_ADMIN+"')") public void scoreVerticals() throws InvalidParameterException, IOException, InterruptedException { aggregationFacadeService.scoreAll(); } @GetMapping("/aggregate/verticals") - @Operation(summary="Launch sanitisation of verticals ID") - public void sanitizeVertical() throws InvalidParameterException, IOException { + @Operation(summary="Launch aggregation of all verticals") + @PreAuthorize("hasAuthority('"+RolesConstants.ROLE_ADMIN+"')") + public void sanitizeAllVertical() throws InvalidParameterException, IOException { aggregationFacadeService.sanitizeAllVerticals(); } + + @GetMapping("/aggregate/verticals/{vertical}") + @Operation(summary="Launch aggregation of a specific vertical") + @PreAuthorize("hasAuthority('"+RolesConstants.ROLE_ADMIN+"')") + public void sanitizeSpecificVertical(@PathVariable String vertical) throws InvalidParameterException, IOException { + aggregationFacadeService.sanitizeVertical(verticalConfigService.getConfigById(vertical)); + } + - @GetMapping("/aggregate") - @Operation(summary="Launch sanitisation of all products") + @GetMapping("/aggregate/products") + @Operation(summary="Launch aggregation of all products") + @PreAuthorize("hasAuthority('"+RolesConstants.ROLE_ADMIN+"')") public void sanitize() throws InvalidParameterException, IOException { aggregationFacadeService.sanitizeAll(); } - @GetMapping("/aggregate/{gtin}") + @GetMapping("/aggregate/gtin/{gtin}") @Operation(summary="Launch sanitisation of a given products") + @PreAuthorize("hasAuthority('"+RolesConstants.ROLE_ADMIN+"')") public void sanitizeOne(@PathVariable Long gtin ) throws InvalidParameterException, IOException, ResourceNotFoundException, AggregationSkipException { aggregationFacadeService.sanitizeAndSave(repository.getById(gtin)); } diff --git a/api/src/main/java/org/open4goods/api/services/aggregation/services/realtime/AttributeRealtimeAggregationService.java b/api/src/main/java/org/open4goods/api/services/aggregation/services/realtime/AttributeRealtimeAggregationService.java index 432b570a5..ae0a40e3c 100644 --- a/api/src/main/java/org/open4goods/api/services/aggregation/services/realtime/AttributeRealtimeAggregationService.java +++ b/api/src/main/java/org/open4goods/api/services/aggregation/services/realtime/AttributeRealtimeAggregationService.java @@ -1,10 +1,8 @@ package org.open4goods.api.services.aggregation.services.realtime; -import java.util.Collection; import java.util.HashMap; -import java.util.List; +import java.util.HashSet; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.stream.Collectors; @@ -20,7 +18,6 @@ import org.open4goods.commons.model.attribute.Attribute; import org.open4goods.commons.model.constants.ReferentielKey; import org.open4goods.commons.model.data.DataFragment; -import org.open4goods.commons.model.product.AggregatedFeature; import org.open4goods.commons.model.product.IndexedAttribute; import org.open4goods.commons.model.product.Product; import org.open4goods.commons.model.product.ProductAttribute; @@ -53,30 +50,33 @@ public void onProduct(Product data, VerticalConfig vConf) throws AggregationSkip ////////////////////////////////////////// // Remove excluded attributes - // TODO / Usefull for batch mode, could remove once initial sanitization + // TODO(p3,perf) / Usefull for batch mode, could remove once initial sanitization vConf.getAttributesConfig().getExclusions().forEach(e -> { data.getAttributes().getAll().remove(e); }); - // NOTE(p3,design) : Remove, this is some kind of legacy bug, some attributes have uncoherent and too many sources. Hope this is a done bug.. - // data.getAttributes().getAll().keySet().removeIf(e-> data.getAttributes().getAll().get(e).sourcesCount() > 10); - - ////////////////////////////////////////////////////////////////////////// - // Checking if all mandatory attributes are present for this product - ////////////////////////////////////////////////////////////////////////// -// if (!data.getAttributes().getAggregatedAttributes().keySet().containsAll(vConf.getAttributesConfig().getMandatory())) { -// // Missing attributes. -// -// Set missing = vConf.getAttributesConfig().getMandatory(); -// missing.removeAll(data.getAttributes().getAggregatedAttributes().keySet()); -// -// dedicatedLogger.warn("{} excluded from {}. Missing mandatory attributes : {}", data.getId(), vConf.getId(), missing); -// data.setExcluded(true); -// } else { -// data.setExcluded(false); -// } + ///////////////////////////////////////////////// + // Cleaning brands + // NOTE : Should be disabled after recovery batch, but need to be run each time to + // take in account modifications of configurable brandAlias() and brandExclusions() + ///////////////////////////////////////////////// + + String actualBrand = data.brand(); + Map akaBrands = new HashMap<>(data.getAkaBrands()); + + data.getAttributes().getReferentielAttributes().remove(ReferentielKey.BRAND); + data.akaBrands().clear(); + data.addBrand(null, actualBrand, vConf.getBrandsExclusion(), vConf.getBrandsAlias()); + + akaBrands.entrySet().forEach(e -> { + data.addBrand(e.getKey(), e.getValue(), vConf.getBrandsExclusion(), vConf.getBrandsAlias()); + }); + + + + // Attributing taxomy to attributes data.getAttributes().getAll().values().forEach(a -> { Set icecatTaxonomyIds = featureService.resolveFeatureName(a.getName()); @@ -98,14 +98,8 @@ public void onProduct(Product data, VerticalConfig vConf) throws AggregationSkip for (ProductAttribute attr : data.getAttributes().getAll().values()) { // Checking if a potential AggregatedAttribute - // TODO(P1) : Detect and parse from the icecat taxonomy - AttributeConfig attrConfig = attributesConfig.resolveFromProductAttribute(attr); - - - //String indexedName = attributesConfig.indexedAttributeNameOrNull(attr); - // We have a "raw" attribute that matches an aggregationconfig if (null != attrConfig) { @@ -171,7 +165,7 @@ private boolean shouldExclude(Product data, VerticalConfig vConf) { } Set attrKeys = data.getAttributes().getattributesAsStringKeys(); - if (!attrKeys.containsAll(vConf.getRequiredAttributes())) { + if (vConf.getRequiredAttributes() != null && !attrKeys.containsAll(vConf.getRequiredAttributes())) { dedicatedLogger.warn("Excluded because attributes are missing : {}", data ); return true; } @@ -179,19 +173,7 @@ private boolean shouldExclude(Product data, VerticalConfig vConf) { return false; } - - // EXTRACTING FEATURES ATTRIBUTES - - - - - - - - - - - + /** * On data fragment agg leveln we increment the "all" field, with sourced values @@ -204,10 +186,6 @@ private boolean shouldExclude(Product data, VerticalConfig vConf) { @Override public Map onDataFragment(final DataFragment dataFragment, final Product product, VerticalConfig vConf) throws AggregationSkipException { - if (dataFragment.getAttributes().size() == 0) { - return null; - } - try { // AttributesConfig attributesConfig = vConf.getAttributesConfig(); @@ -244,48 +222,12 @@ public Map onDataFragment(final DataFragment dataFragment, final ///////////////////////////////////////// // Update referentiel attributes ///////////////////////////////////////// - handleReferentielAttributes(dataFragment, product); + handleReferentielAttributes(dataFragment, product, vConf); // TODO : Add BRAND / MODEL from matches from attributes - ////////////////////////// - // Aggregating unmatched attributes - /////////////////////////// - -// for (Attribute attr : dataFragment.getAttributes()) { -// // Checking if to be removed -//// if (toRemoveFromUnmatched.contains(attr.getName())) { -//// continue; -//// } -// -// // TODO : remove from a config list -// -// ProductAttribute agg = product.getAttributes().getUnmapedAttributes().stream().filter(e -> e.getName().equals(attr.getName())).findAny().orElse(null); -// -// if (null == agg) { -// // A first time match -// agg = new ProductAttribute(); -// agg.setName(attr.getName()); -// } -// agg.addAttribute(attr, new UnindexedKeyValTimestamp(dataFragment.getDatasourceName(), attr.getValue())); -// -// product.getAttributes().getUnmapedAttributes().add(agg); -// } - - // Removing -// product.getAttributes().setUnmapedAttributes(product.getAttributes().getUnmapedAttributes().stream() -// // TODO : Should be from path -// // TODO : apply from sanitisation -// .filter(e -> !e.getName().contains("CATEGORY")) -//// .filter(e -> !toRemoveFromUnmatched.contains(e.getName())) -// .collect(Collectors.toSet())); - - // TODO : Removing matchlist again to handle remove of old attributes in case of - // configuration change -// product.getAttributes().getUnmapedAttributes(). + } catch (Exception e) { - // TODO dedicatedLogger.error("Unexpected error", e); - e.printStackTrace(); } onProduct(product, vConf); @@ -375,49 +317,48 @@ private boolean isFeatureAttribute(Attribute e, AttributesConfig attributesConfi * @param output The product output to which referential attributes are to be * added or updated. */ - private void handleReferentielAttributes(DataFragment fragment, Product output) { - for (Entry attr : fragment.getReferentielAttributes().entrySet()) { - ReferentielKey key = attr.getKey(); - String value = attr.getValue().toUpperCase(); - - if (StringUtils.isEmpty(value)) { - continue; - } - - String existing = output.getAttributes().getReferentielAttributes().get(key); + private void handleReferentielAttributes(DataFragment fragment, Product output, VerticalConfig vConf) { + + /////////////////////// + // Adding brand + /////////////////////// + String brand = fragment.brand(); + if (!StringUtils.isEmpty(brand)) { + output.addBrand(fragment.getDatasourceName(), brand, vConf.getBrandsExclusion(), vConf.getBrandsAlias()); + } + + + /////////////////////// + // Adding model + /////////////////////// + String model = fragment.getReferentielAttributes().get(ReferentielKey.MODEL); + if (!StringUtils.isEmpty(model)) { + output.addModel(model); + } - if (StringUtils.isEmpty(existing)) { - output.getAttributes().addReferentielAttribute(key, value); - } else if (!existing.equals(value)) { - // We have a variation ! - switch (key) { - case MODEL: - dedicatedLogger.info("Adding different {} name as alternate id. Existing is {}, would have erased with {}", key, existing, value); - output.addModel(value); - break; - case BRAND: - output.getAkaBrands().put(fragment.getDatasourceName(), value); - break; - case GTIN: - if (value != null && !existing.equals(value)) { - try { - long existingGtin = Long.parseLong(existing); - long newGtin = Long.parseLong(value); - if (existingGtin != newGtin) { - dedicatedLogger.error("Overriding GTIN from {} to {}", existing, newGtin); - output.getAttributes().getReferentielAttributes().put(ReferentielKey.GTIN, value); - } - } catch (NumberFormatException e) { - dedicatedLogger.error("Invalid GTIN format: existing = {}, new = {}", existing, value, e); - } - } - break; - default: - dedicatedLogger.warn("Skipping referential attribute erasure for {}. Existing is {}, would have erased with {}", key, existing, value); - break; + /////////////////////// + // Handling gtin (NOTE : useless since gtin is used as ID, so coupling is done previously + /////////////////////// + String gtin = fragment.gtin(); + if (!StringUtils.isEmpty(gtin)) { + String existing = output.gtin(); + + if (!existing.equals(gtin)) { + try { + long existingGtin = Long.parseLong(existing); + long newGtin = Long.parseLong(gtin); + if (existingGtin != newGtin) { + dedicatedLogger.error("Overriding GTIN from {} to {}", existing, newGtin); + output.getAttributes().getReferentielAttributes().put(ReferentielKey.GTIN, gtin); + } + } catch (NumberFormatException e) { + dedicatedLogger.error("Invalid GTIN format: existing = {}, new = {}", existing, gtin, e); } } + } + + } /** diff --git a/api/src/main/java/org/open4goods/api/services/store/DataFragmentStoreService.java b/api/src/main/java/org/open4goods/api/services/store/DataFragmentStoreService.java index 2eff8c770..e56cae9ee 100644 --- a/api/src/main/java/org/open4goods/api/services/store/DataFragmentStoreService.java +++ b/api/src/main/java/org/open4goods/api/services/store/DataFragmentStoreService.java @@ -199,7 +199,7 @@ public void aggregateAndstore(Collection buffer) { // Fastening, by checking if exaclty same datafragment than previously Long hash = data.getDatasourceCodes().get(df.getDatasourceName()); - if (null != hash && hash.equals(df.getFragmentHashCode())) { + if (null != hash && hash.equals(Long.valueOf(df.getFragmentHashCode()))) { // We can proceed to partial update, we just update lasttimechange and prices logger.info("Proceeding to partial update for {}",data.getId()); // Updating price diff --git a/commons/src/main/java/org/open4goods/commons/config/yml/ui/AttributesConfig.java b/commons/src/main/java/org/open4goods/commons/config/yml/ui/AttributesConfig.java index 398627c29..4a90c9e60 100644 --- a/commons/src/main/java/org/open4goods/commons/config/yml/ui/AttributesConfig.java +++ b/commons/src/main/java/org/open4goods/commons/config/yml/ui/AttributesConfig.java @@ -21,13 +21,6 @@ public class AttributesConfig { - /** - * The mandatoryattributes - */ - @JsonMerge - private Set mandatory = new HashSet<>(); - - /** * The specific configs configurations */ @@ -145,6 +138,11 @@ public Map> synonyms() { } + /** + * Resolve a product attribute against icecat taxonomy or faalback on attributes config + * @param attr + * @return + */ public AttributeConfig resolveFromProductAttribute(ProductAttribute attr) { AttributeConfig ret = null; @@ -278,15 +276,5 @@ public void setExclusions(Set exclusions) { } - public Set getMandatory() { - return mandatory; - } - - - public void setMandatory(Set mandatory) { - this.mandatory = mandatory; - } - - } diff --git a/commons/src/main/java/org/open4goods/commons/config/yml/ui/VerticalConfig.java b/commons/src/main/java/org/open4goods/commons/config/yml/ui/VerticalConfig.java index af6e23ed4..2c67622b7 100644 --- a/commons/src/main/java/org/open4goods/commons/config/yml/ui/VerticalConfig.java +++ b/commons/src/main/java/org/open4goods/commons/config/yml/ui/VerticalConfig.java @@ -103,6 +103,7 @@ public class VerticalConfig{ /** * The attributes that must be present. If not, the product will have excluded set to true */ + @JsonMerge private Set requiredAttributes = new HashSet(); /** @@ -111,10 +112,25 @@ public class VerticalConfig{ @JsonMerge private CommentsAggregationConfig commentsConfig = new CommentsAggregationConfig(); + /** + * The company (sustainalytics form) that operates the brand + */ @JsonMerge private Map brandsCompanyMapping = new HashMap<>(); + @JsonMerge + /** + * Brand alias mappings (eg : LG ELECTRONICS : LG) + */ + private Map brandsAlias = new HashMap<>(); + + /** + * The brands that must be removes (eg. NON COMMUNIQUE) + */ + private Set brandsExclusion = new HashSet(); + + /** * The I18n URL Mappings. Think SEO ! */ @@ -728,6 +744,22 @@ public void setRequiredAttributes(Set requiredAttributes) { this.requiredAttributes = requiredAttributes; } + public Map getBrandsAlias() { + return brandsAlias; + } + + public void setBrandsAlias(Map brandsAlias) { + this.brandsAlias = brandsAlias; + } + + public Set getBrandsExclusion() { + return brandsExclusion; + } + + public void setBrandsExclusion(Set brandsExclusion) { + this.brandsExclusion = brandsExclusion; + } + diff --git a/commons/src/main/java/org/open4goods/commons/model/product/Product.java b/commons/src/main/java/org/open4goods/commons/model/product/Product.java index ed7cdf282..eb2e22dee 100644 --- a/commons/src/main/java/org/open4goods/commons/model/product/Product.java +++ b/commons/src/main/java/org/open4goods/commons/model/product/Product.java @@ -98,7 +98,7 @@ public class Product implements Standardisable { private Set akaModels = new HashSet<>(); - /** The list of other id's known for this product **/ + /** The list of other brands known for this product **/ private Map akaBrands = new HashMap(); @@ -611,14 +611,18 @@ public void addModel(String value) { String model = StringUtils.normalizeSpace(value).toUpperCase(); // TODO(conf,p2) : Eviction size from conf - if (StringUtils.isEmpty(value) || value.length() < 3) { + if (StringUtils.isEmpty(model) || model.length() < 3) { return; } // Splitting on conventionnal suffixes (/ - .) // TODO(conf,p2) : splitters from Const / conf String[]frags = model.split("/|\\|.|-"); - akaModels.add(value); + + if (!model.equals(model())) { + akaModels.add(model); + } + if (frags.length > 1) { logger.info("Found an alternative model : " + frags[0]); akaModels.add(frags[0]); @@ -638,6 +642,42 @@ public void addModel(String value) { } } + /** + * Add the brand referentiel attribute, applying some sanitisation mechanism + * @param value + */ + public void addBrand(String datasource, String value, Set brandExclusions, Map brandsMappings) { + + if (StringUtils.isEmpty(value)) { + return; + } + + String brand = StringUtils.stripAccents(StringUtils.normalizeSpace(value)).toUpperCase(); + + if (null != brandExclusions && brandExclusions.contains(brand)) { + logger.info("Excluding brand {}", brand); + return; + } + + if (null != brandsMappings) { + String replacement = brandsMappings.get(brand); + if (null != replacement) { + brand = replacement; + logger.info("replacing brand {} with {}", brand, replacement); + } + } + + + if (StringUtils.isEmpty(brand()) ) { + attributes.addReferentielAttribute(ReferentielKey.BRAND, brand); + } else { + if (!akaBrands.values().contains(brand)) { + akaBrands.put(datasource, brand); + } + + } + } + /** * * @return the shortest model name diff --git a/commons/src/main/java/org/open4goods/commons/services/SearchService.java b/commons/src/main/java/org/open4goods/commons/services/SearchService.java index 67b59d117..aa888d66a 100644 --- a/commons/src/main/java/org/open4goods/commons/services/SearchService.java +++ b/commons/src/main/java/org/open4goods/commons/services/SearchService.java @@ -135,7 +135,7 @@ public VerticalSearchResponse globalSearch(String initialQuery, Integer fromPric * @param request * @return */ -// @Cacheable(keyGenerator = CacheConstants.KEY_GENERATOR, cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) + @Cacheable(keyGenerator = CacheConstants.KEY_GENERATOR, cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) public VerticalSearchResponse verticalSearch(VerticalConfig vertical, VerticalSearchRequest request) { VerticalSearchResponse vsr = new VerticalSearchResponse(); diff --git a/verticals/src/main/resources/verticals/_default.yml b/verticals/src/main/resources/verticals/_default.yml index 98d431cac..ddd49c513 100644 --- a/verticals/src/main/resources/verticals/_default.yml +++ b/verticals/src/main/resources/verticals/_default.yml @@ -102,7 +102,26 @@ descriptionsAggregationConfig: globalTechnicalFilters: - "COLOR" + + + + + + + +#################################################################################### +# Brands configuration +#################################################################################### +brandsExclusion: + - NON COMMUNIQUE + - GENERIQUE + +#brandsMappings: +# RR : RR + + + #################################################################################### # ATTRIBUTES CONFIGURATION #################################################################################### @@ -196,7 +215,7 @@ attributesConfig: - OUI - Y - + #################################################################################### # ATTRIBUTES DEFINITIONS diff --git a/verticals/src/main/resources/verticals/bouilloire.yml b/verticals/src/main/resources/verticals/bouilloire.yml index 1b0e8e256..d8f6c4779 100644 --- a/verticals/src/main/resources/verticals/bouilloire.yml +++ b/verticals/src/main/resources/verticals/bouilloire.yml @@ -179,10 +179,6 @@ brandsCompanyMapping: #################################################################################### attributesConfig: # If one of the following attribute is missing on a product, the product will be disassociated from this vertical. (setting vertical to null) -# mandatory: -# - "DIAGONALE_POUCES" -# - "REPAIRABILITY_INDEX" -# - "TV_TYPE" #################################################################################### # ATTRIBUTES DEFINITIONS diff --git a/verticals/src/main/resources/verticals/tv.yml b/verticals/src/main/resources/verticals/tv.yml index dadb4d2e9..2aa651c05 100644 --- a/verticals/src/main/resources/verticals/tv.yml +++ b/verticals/src/main/resources/verticals/tv.yml @@ -218,10 +218,6 @@ brandsCompanyMapping: #################################################################################### attributesConfig: # If one of the following attribute is missing on a product, the product will be disassociated from this vertical. (setting vertical to null) -# mandatory: -# - "DIAGONALE_POUCES" -# - "REPAIRABILITY_INDEX" -# - "TV_TYPE" #################################################################################### # ATTRIBUTES DEFINITIONS