From c4976f0fd8d759d615dafa7d4465e78b42ef247f Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 2 Aug 2024 12:05:18 +0200 Subject: [PATCH 01/52] Separate processing of ISBN and GTIN datasets in opendata export --- .../ui/services/OpenDataService.java | 56 +++++++++---------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index bf08fc5cd..a0621efc4 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -23,6 +23,7 @@ import org.open4goods.dao.ProductRepository; import org.open4goods.exceptions.TechnicalException; import org.open4goods.helper.ThrottlingInputStream; +import org.open4goods.model.BarcodeType; import org.open4goods.model.constants.CacheConstants; import org.open4goods.model.product.Product; import org.open4goods.ui.config.yml.UiConfig; @@ -117,57 +118,54 @@ public void generateOpendata() { if (exportRunning.get()) { LOGGER.error("Opendata export is already running"); - } else { - exportRunning.set(true); + return; } + exportRunning.set(true); + ZipOutputStream zos = null; - try { + FileOutputStream fos = null; + try { uiConfig.tmpOpenDataFile().getParentFile().mkdirs(); - FileOutputStream fos = new FileOutputStream(uiConfig.tmpOpenDataFile()); + fos = new FileOutputStream(uiConfig.tmpOpenDataFile()); zos = new ZipOutputStream(fos); - String filename = "open4goods-full-gtin-dataset.csv"; - ZipEntry entry = new ZipEntry(filename); // create a zip entry and add it pageSize ZipOutputStream - zos.putNextEntry(entry); - - CSVWriter writer = new CSVWriter(new OutputStreamWriter(zos)); - - // Writing the header - writer.writeNext(header); - - // Fixing the count - AtomicLong count = new AtomicLong(); + // Process ISBN_13 + LOGGER.info("Starting process for ISBN_13"); + processAndAddToZip(zos, "open4goods-isbn-dataset.csv", BarcodeType.ISBN_13); - // Iterating on datas - LOGGER.info("Starting opendata export"); - aggregatedDataRepository.exportAll().forEach(e -> { - count.incrementAndGet(); - writer.writeNext(toEntry(e)); // write the contents - }); + // Process GTIN/EAN (excluding ISBN_13) + LOGGER.info("Starting process for GTIN/EAN excluding ISBN_13"); + processAndAddToZip(zos, "open4goods-gtin-dataset.csv", BarcodeType.ISBN_13, true); - - - writer.flush(); - zos.closeEntry(); + zos.close(); + fos.close(); // Moving the tmp file - FileUtils.deleteQuietly(uiConfig.openDataFile()); + if (uiConfig.openDataFile().exists()) { + FileUtils.deleteQuietly(uiConfig.openDataFile()); + } FileUtils.moveFile(uiConfig.tmpOpenDataFile(), uiConfig.openDataFile()); - exportRunning.set(false); - LOGGER.info("{} rows exported in opendata CSV file located at {}", count.get(), - uiConfig.openDataFile().getAbsolutePath()); + LOGGER.info("Opendata CSV files generated and zipped successfully."); } catch (Exception e) { LOGGER.error("Error while generating opendata set", e); } finally { IOUtils.closeQuietly(zos); + IOUtils.closeQuietly(fos); + exportRunning.set(false); } } + private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType) throws IOException { + processAndAddToZip(zos, filename, barcodeType, false); + } + + private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType, boolean invertCondition) throws IOException {} + /** * Convert an aggregateddata pageSize a csv row * From a1965b97ab5d56604881d8c72e027facb57258e8 Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 2 Aug 2024 12:09:12 +0200 Subject: [PATCH 02/52] Implement processAndAddToZip method --- .../ui/services/OpenDataService.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index a0621efc4..a123f111c 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -164,7 +164,27 @@ private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeTyp processAndAddToZip(zos, filename, barcodeType, false); } - private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType, boolean invertCondition) throws IOException {} + private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType, boolean invertCondition) throws IOException { + ZipEntry entry = new ZipEntry(filename); + zos.putNextEntry(entry); + CSVWriter writer = new CSVWriter(new OutputStreamWriter(zos)); + writer.writeNext(header); + + AtomicLong count = new AtomicLong(); + try { + aggregatedDataRepository.exportAll().filter(e -> + invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) + ).forEach(e -> { + count.incrementAndGet(); + writer.writeNext(toEntry(e)); + }); + writer.flush(); + zos.closeEntry(); + LOGGER.info("{} rows exported in {}.", count.get(), filename); + } catch (Exception e) { + LOGGER.error("Error during processing of {}: {}", filename, e.getMessage()); + } + } /** * Convert an aggregateddata pageSize a csv row From 459ffedd689a3a8b8988fa822bf176b523a8a6dd Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 2 Aug 2024 17:54:27 +0200 Subject: [PATCH 03/52] Typo --- ui/src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/main/resources/application.yml b/ui/src/main/resources/application.yml index 2c2e68c3a..436eb6ab7 100644 --- a/ui/src/main/resources/application.yml +++ b/ui/src/main/resources/application.yml @@ -139,7 +139,7 @@ brandConfig: imageGenerationConfig: prompt: | Create a simple icon of a {VERTICAL} that strictly adheres to the following guidelines: - - The background color must be fully (#FFFFFF). + - The background color must be fully white (#FFFFFF). - The white background does not contains any gradient or texture. - The icon should be flat, in a modern, minimalist style. - The lines must be clean and simple. From c377cc1a2c9323c23a8c04b9a010ea4822e258a0 Mon Sep 17 00:00:00 2001 From: scezen Date: Sun, 4 Aug 2024 15:00:03 +0200 Subject: [PATCH 04/52] Small fixes, comments --- .../ui/services/OpenDataService.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index a123f111c..ec3eec847 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -63,13 +63,14 @@ public class OpenDataService { private AtomicInteger concurrentDownloads = new AtomicInteger(0); - - + private static final String ISBN_DATASET_FILENAME = "open4goods-isbn-dataset.csv"; + private static final String GTIN_DATASET_FILENAME = "open4goods-gtin-dataset.csv"; @Autowired - public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig) { + public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig){ this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; + generateOpendata(); } /** @@ -81,10 +82,10 @@ public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiCo */ public InputStream limitedRateStream() throws TechnicalException, FileNotFoundException { - // TODO : pageNumber conf + // TODO : in conf RateLimiter rateLimiter = RateLimiter.create(DOWNLOAD_SPEED_KB * FileUtils.ONE_KB); - // TODO : pageNumber conf + // TODO : in conf if (concurrentDownloads.get() >= CONCURRENT_DOWNLOADS) { throw new TechnicalException("Too many requests "); } else { @@ -109,11 +110,11 @@ public void close() throws IOException { } /** - * Iterates over all aggregatedData pageSize generate the zipped opendata CSV file + * Iterates over all aggregated data to generate the zipped opendata CSV file. * - * TODO : Schedule pageNumber conf + * TODO : Schedule in conf */ - @Scheduled(initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) + //@Scheduled(initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) public void generateOpendata() { if (exportRunning.get()) { @@ -134,11 +135,11 @@ public void generateOpendata() { // Process ISBN_13 LOGGER.info("Starting process for ISBN_13"); - processAndAddToZip(zos, "open4goods-isbn-dataset.csv", BarcodeType.ISBN_13); + processAndAddToZip(zos, ISBN_DATASET_FILENAME, BarcodeType.ISBN_13); // Process GTIN/EAN (excluding ISBN_13) - LOGGER.info("Starting process for GTIN/EAN excluding ISBN_13"); - processAndAddToZip(zos, "open4goods-gtin-dataset.csv", BarcodeType.ISBN_13, true); + LOGGER.info("Starting process for GTIN/EAN"); + processAndAddToZip(zos, GTIN_DATASET_FILENAME, BarcodeType.ISBN_13, true); zos.close(); fos.close(); @@ -164,6 +165,11 @@ private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeTyp processAndAddToZip(zos, filename, barcodeType, false); } + /** + * Processes the data and adds it to the zip output stream. + * + * @param invertCondition whether to invert the condition for filtering the data + */ private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType, boolean invertCondition) throws IOException { ZipEntry entry = new ZipEntry(filename); zos.putNextEntry(entry); @@ -208,7 +214,6 @@ private String[] toEntry(Product data) { line[4] = String.valueOf(data.getLastChange()); // "gs1_country" line[5] = data.getGtinInfos().getCountry(); - // "upcType" line[6] = data.getGtinInfos().getUpcType().toString(); From 1afacd79c0d73ef8335fd5e27e2424a791825463 Mon Sep 17 00:00:00 2001 From: scezen Date: Sun, 4 Aug 2024 15:03:56 +0200 Subject: [PATCH 05/52] Re-enable scheduling --- .../main/java/org/open4goods/ui/services/OpenDataService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index ec3eec847..2892236c8 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -70,7 +70,6 @@ public class OpenDataService { public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig){ this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; - generateOpendata(); } /** @@ -114,7 +113,7 @@ public void close() throws IOException { * * TODO : Schedule in conf */ - //@Scheduled(initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) + @Scheduled(initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) public void generateOpendata() { if (exportRunning.get()) { From bc3182d6719d75d45fc1b7a83d0169d2011392c1 Mon Sep 17 00:00:00 2001 From: scezen Date: Sun, 4 Aug 2024 22:11:16 +0200 Subject: [PATCH 06/52] Separate directory preparation from generateOpendata --- .../ui/services/OpenDataService.java | 41 ++++--------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 2892236c8..bf79d5469 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -115,47 +115,19 @@ public void close() throws IOException { */ @Scheduled(initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) public void generateOpendata() { - - if (exportRunning.get()) { + if (exportRunning.getAndSet(true)) { LOGGER.error("Opendata export is already running"); return; } - exportRunning.set(true); - - ZipOutputStream zos = null; - FileOutputStream fos = null; - try { - uiConfig.tmpOpenDataFile().getParentFile().mkdirs(); - - fos = new FileOutputStream(uiConfig.tmpOpenDataFile()); - zos = new ZipOutputStream(fos); - - // Process ISBN_13 - LOGGER.info("Starting process for ISBN_13"); - processAndAddToZip(zos, ISBN_DATASET_FILENAME, BarcodeType.ISBN_13); - - // Process GTIN/EAN (excluding ISBN_13) - LOGGER.info("Starting process for GTIN/EAN"); - processAndAddToZip(zos, GTIN_DATASET_FILENAME, BarcodeType.ISBN_13, true); - - zos.close(); - fos.close(); - - // Moving the tmp file - if (uiConfig.openDataFile().exists()) { - FileUtils.deleteQuietly(uiConfig.openDataFile()); - } - FileUtils.moveFile(uiConfig.tmpOpenDataFile(), uiConfig.openDataFile()); - + prepareDirectories(); + processDataFiles(); + moveTmpFilesToFinalDestination(); LOGGER.info("Opendata CSV files generated and zipped successfully."); - } catch (Exception e) { LOGGER.error("Error while generating opendata set", e); } finally { - IOUtils.closeQuietly(zos); - IOUtils.closeQuietly(fos); exportRunning.set(false); } } @@ -164,6 +136,11 @@ private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeTyp processAndAddToZip(zos, filename, barcodeType, false); } + private void prepareDirectories() throws IOException { + uiConfig.tmpIsbnZipFile().getParentFile().mkdirs(); + uiConfig.tmpGtinZipFile().getParentFile().mkdirs(); + } + /** * Processes the data and adds it to the zip output stream. * From ab3fe41a1a82595faab6e6d1d6e56cb567273c76 Mon Sep 17 00:00:00 2001 From: scezen Date: Sun, 4 Aug 2024 22:12:33 +0200 Subject: [PATCH 07/52] Extract file moving logic to separate method --- .../ui/services/OpenDataService.java | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index bf79d5469..40caaefda 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -1,12 +1,6 @@ package org.open4goods.ui.services; -import java.io.BufferedInputStream; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStreamWriter; +import java.io.*; import java.text.CharacterIterator; import java.text.StringCharacterIterator; import java.time.Instant; @@ -141,6 +135,27 @@ private void prepareDirectories() throws IOException { uiConfig.tmpGtinZipFile().getParentFile().mkdirs(); } + private void processDataFiles() throws IOException { + LOGGER.info("Starting process for ISBN_13"); + processAndCreateZip(ISBN_DATASET_FILENAME, BarcodeType.ISBN_13, uiConfig.tmpIsbnZipFile()); + + LOGGER.info("Starting process for GTIN/EAN"); + processAndCreateZip(GTIN_DATASET_FILENAME, BarcodeType.ISBN_13, uiConfig.tmpGtinZipFile(), true); + } + + private void moveTmpFilesToFinalDestination() throws IOException { + moveFile(uiConfig.tmpIsbnZipFile(), uiConfig.isbnZipFile()); + moveFile(uiConfig.tmpGtinZipFile(), uiConfig.gtinZipFile()); + } + + private void moveFile(File src, File dest) throws IOException { + if (dest.exists()) { + FileUtils.deleteQuietly(dest); + } + FileUtils.moveFile(src, dest); + } + + /** * Processes the data and adds it to the zip output stream. * From b3b30118429f876a0a189893ec83635ea76d86b5 Mon Sep 17 00:00:00 2001 From: scezen Date: Sun, 4 Aug 2024 22:13:53 +0200 Subject: [PATCH 08/52] Use try-with-resources for automatic resource management --- .../ui/services/OpenDataService.java | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 40caaefda..c98f59c8a 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -126,8 +126,8 @@ public void generateOpendata() { } } - private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType) throws IOException { - processAndAddToZip(zos, filename, barcodeType, false); + private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile) throws IOException { + processAndCreateZip(filename, barcodeType, zipFile, false); } private void prepareDirectories() throws IOException { @@ -156,28 +156,25 @@ private void moveFile(File src, File dest) throws IOException { } - /** - * Processes the data and adds it to the zip output stream. - * - * @param invertCondition whether to invert the condition for filtering the data - */ - private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType, boolean invertCondition) throws IOException { - ZipEntry entry = new ZipEntry(filename); - zos.putNextEntry(entry); - CSVWriter writer = new CSVWriter(new OutputStreamWriter(zos)); - writer.writeNext(header); + private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile, boolean invertCondition) throws IOException { + try (FileOutputStream fos = new FileOutputStream(zipFile); + ZipOutputStream zos = new ZipOutputStream(fos); + CSVWriter writer = new CSVWriter(new OutputStreamWriter(zos))) { - AtomicLong count = new AtomicLong(); - try { - aggregatedDataRepository.exportAll().filter(e -> + ZipEntry entry = new ZipEntry(filename); + zos.putNextEntry(entry); + writer.writeNext(header); + + AtomicLong count = new AtomicLong(); + aggregatedDataRepository.exportAll().limit(500).filter(e -> invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) ).forEach(e -> { count.incrementAndGet(); writer.writeNext(toEntry(e)); }); - writer.flush(); - zos.closeEntry(); LOGGER.info("{} rows exported in {}.", count.get(), filename); + + zos.closeEntry(); } catch (Exception e) { LOGGER.error("Error during processing of {}: {}", filename, e.getMessage()); } From 6c0a05a403d9fc188bf3a8db0509f1d111ec4836 Mon Sep 17 00:00:00 2001 From: scezen Date: Sun, 4 Aug 2024 22:18:16 +0200 Subject: [PATCH 09/52] Update OpenDataController + UiConfig --- .../open4goods/ui/config/yml/UiConfig.java | 15 +++++++++++-- .../ui/pages/OpenDataController.java | 21 +++++++++++++++++++ .../ui/services/OpenDataService.java | 3 +-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/config/yml/UiConfig.java b/ui/src/main/java/org/open4goods/ui/config/yml/UiConfig.java index 9ecfc9fd9..87acc3677 100644 --- a/ui/src/main/java/org/open4goods/ui/config/yml/UiConfig.java +++ b/ui/src/main/java/org/open4goods/ui/config/yml/UiConfig.java @@ -251,10 +251,21 @@ public File openDataFile() { return new File(rootFolder + File.separator+"opendata"+File.separator+"full.zip"); } - public File tmpOpenDataFile() { - return new File(rootFolder + File.separator+"opendata"+File.separator+"full-tmp.zip"); + public File isbnZipFile() { + return new File(rootFolder + File.separator + "opendata" + File.separator + "isbn.zip"); } + public File tmpIsbnZipFile() { + return new File(rootFolder + File.separator + "opendata" + File.separator + "isbn-tmp.zip"); + } + + public File gtinZipFile() { + return new File(rootFolder + File.separator + "opendata" + File.separator + "gtin.zip"); + } + + public File tmpGtinZipFile() { + return new File(rootFolder + File.separator + "opendata" + File.separator + "gtin-tmp.zip"); + } public String getRootFolder() { return rootFolder; diff --git a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java index 377f44589..179dceb5e 100644 --- a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java +++ b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java @@ -2,6 +2,8 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; +import java.util.List; import org.apache.commons.io.IOUtils; import org.open4goods.exceptions.TechnicalException; @@ -53,6 +55,15 @@ public OpenDataController(OpenDataService openDataService) { public SitemapEntry getExposedUrls() { return SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, DEFAULT_PATH, 0.3, ChangeFreq.YEARLY); } + + @Override + public List getMultipleExposedUrls() { + return Arrays.asList( + SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, DEFAULT_PATH, 0.3, ChangeFreq.YEARLY), + SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/gtin-open-data.zip", 0.3, ChangeFreq.YEARLY), + SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/isbn-open-data.zip", 0.3, ChangeFreq.YEARLY) + ); + } @GetMapping(value = {DEFAULT_PATH}) public ModelAndView opendata(final HttpServletRequest request) { @@ -64,6 +75,16 @@ public ModelAndView opendata(final HttpServletRequest request) { return ret; } + @GetMapping(path = "/opendata/gtin-open-data.zip") + public void downloadGtinData(final HttpServletResponse response) throws IOException { + + } + + @GetMapping(path = "/opendata/isbn-open-data.zip") + public void downloadIsbnData(final HttpServletResponse response) throws IOException { + + } + @GetMapping(path = "/opendata/gtin-open-data.zip") public void opensearch(final HttpServletResponse response) throws IOException { try (InputStream str = openDataService.limitedRateStream()){ diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index c98f59c8a..e0707cc50 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -155,7 +155,6 @@ private void moveFile(File src, File dest) throws IOException { FileUtils.moveFile(src, dest); } - private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile, boolean invertCondition) throws IOException { try (FileOutputStream fos = new FileOutputStream(zipFile); ZipOutputStream zos = new ZipOutputStream(fos); @@ -166,7 +165,7 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File writer.writeNext(header); AtomicLong count = new AtomicLong(); - aggregatedDataRepository.exportAll().limit(500).filter(e -> + aggregatedDataRepository.exportAll().filter(e -> invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) ).forEach(e -> { count.incrementAndGet(); From ab4a1f1734d6c38b5ebd9fc940efeb7add6b4bda Mon Sep 17 00:00:00 2001 From: scezen Date: Mon, 5 Aug 2024 09:59:16 +0200 Subject: [PATCH 10/52] Add missing cacheable methods for ISBN and GTIN file info --- .../ui/services/OpenDataService.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index e0707cc50..751fbb933 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -233,22 +233,24 @@ public void decrementDownloadCounter() { concurrentDownloads.decrementAndGet(); } - /** - * Return the file size in human readable way - * @return - */ - @Cacheable(key = "#root.method.name", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) - public String fileSize() { - return humanReadableByteCountBin(uiConfig.openDataFile().length()); + @Cacheable(key = "#root.method.name + 'Isbn'", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) + public String isbnFileSize() { + return humanReadableByteCountBin(uiConfig.isbnZipFile().length()); } - /** - * - * @return the last update date - */ - @Cacheable(key = "#root.method.name", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) - public Date lastUpdate() { - return Date.from(Instant.ofEpochMilli(uiConfig.openDataFile().lastModified())); + @Cacheable(key = "#root.method.name + 'Gtin'", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) + public String gtinFileSize() { + return humanReadableByteCountBin(uiConfig.gtinZipFile().length()); + } + + @Cacheable(key = "#root.method.name + 'Isbn'", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) + public Date isbnLastUpdate() { + return Date.from(Instant.ofEpochMilli(uiConfig.isbnZipFile().lastModified())); + } + + @Cacheable(key = "#root.method.name + 'Gtin'", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) + public Date gtinLastUpdate() { + return Date.from(Instant.ofEpochMilli(uiConfig.gtinZipFile().lastModified())); } /** From b69f21b45968f41f79bd15ae256aef37759b7781 Mon Sep 17 00:00:00 2001 From: scezen Date: Mon, 5 Aug 2024 09:59:42 +0200 Subject: [PATCH 11/52] Update OpenDataController to handle multiple file types and improve clarity --- .../ui/pages/OpenDataController.java | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java index 179dceb5e..5d610017a 100644 --- a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java +++ b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java @@ -1,5 +1,7 @@ package org.open4goods.ui.controllers.ui.pages; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; @@ -7,6 +9,7 @@ import org.apache.commons.io.IOUtils; import org.open4goods.exceptions.TechnicalException; +import org.open4goods.ui.config.yml.UiConfig; import org.open4goods.ui.controllers.ui.UiService; import org.open4goods.ui.services.OpenDataService; import org.slf4j.Logger; @@ -38,8 +41,13 @@ public class OpenDataController implements SitemapExposedController{ // The siteConfig private final OpenDataService openDataService; private @Autowired UiService uiService; - public OpenDataController(OpenDataService openDataService) { + private final UiConfig uiConfig; + + + @Autowired + public OpenDataController(OpenDataService openDataService, UiConfig uiConfig) { this.openDataService = openDataService; + this.uiConfig = uiConfig; } /** @@ -64,39 +72,37 @@ public List getMultipleExposedUrls() { SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/isbn-open-data.zip", 0.3, ChangeFreq.YEARLY) ); } - - @GetMapping(value = {DEFAULT_PATH}) + + @GetMapping(value = {DEFAULT_PATH}) public ModelAndView opendata(final HttpServletRequest request) { final ModelAndView ret = uiService.defaultModelAndView("opendata", request); ret.addObject("count", openDataService.totalItems()); - ret.addObject("lastUpdated", openDataService.lastUpdate()); - ret.addObject("fileSize", openDataService.fileSize()); - ret.addObject("page","open data"); + ret.addObject("isbnLastUpdated", openDataService.isbnLastUpdate()); + ret.addObject("isbnFileSize", openDataService.isbnFileSize()); + ret.addObject("gtinLastUpdated", openDataService.gtinLastUpdate()); + ret.addObject("gtinFileSize", openDataService.gtinFileSize()); + ret.addObject("page", "open data"); return ret; } @GetMapping(path = "/opendata/gtin-open-data.zip") public void downloadGtinData(final HttpServletResponse response) throws IOException { - + downloadData(response, "gtin-open-data.zip", uiConfig.gtinZipFile()); } @GetMapping(path = "/opendata/isbn-open-data.zip") public void downloadIsbnData(final HttpServletResponse response) throws IOException { - + downloadData(response, "isbn-open-data.zip", uiConfig.isbnZipFile()); } - @GetMapping(path = "/opendata/gtin-open-data.zip") - public void opensearch(final HttpServletResponse response) throws IOException { - try (InputStream str = openDataService.limitedRateStream()){ + private void downloadData(final HttpServletResponse response, String fileName, File zipFile) throws IOException { + try (InputStream str = new FileInputStream(zipFile)) { response.setHeader("Content-type", "application/octet-stream"); - response.setHeader("Content-Disposition", "attachment; filename=\"gtin-open-data.zip\""); + response.setHeader("Content-Disposition", "attachment; filename=\"" + fileName + "\""); IOUtils.copy(str, response.getOutputStream()); } catch (IOException e) { - LOGGER.error("opendata file download error or interruption : {}",e.getMessage()); + LOGGER.error("opendata file download error or interruption : {}", e.getMessage()); openDataService.decrementDownloadCounter(); - } catch (TechnicalException e) { - response.sendError(429, "Exceding the " + OpenDataService.CONCURRENT_DOWNLOADS + " concurrent downloads availlable"); - LOGGER.error("opendata file download error : {}",e.getMessage()); } } From 2487c517003a3e285c6eedd7e36753b9e2348268 Mon Sep 17 00:00:00 2001 From: scezen Date: Mon, 5 Aug 2024 10:09:07 +0200 Subject: [PATCH 12/52] Fix ZIP entry handling in processAndCreateZip --- .../java/org/open4goods/ui/services/OpenDataService.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 751fbb933..90eb915a6 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -64,6 +64,7 @@ public class OpenDataService { public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig){ this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; + generateOpendata(); } /** @@ -165,15 +166,17 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File writer.writeNext(header); AtomicLong count = new AtomicLong(); - aggregatedDataRepository.exportAll().filter(e -> + aggregatedDataRepository.exportAll().limit(500).filter(e -> invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) ).forEach(e -> { count.incrementAndGet(); writer.writeNext(toEntry(e)); }); + writer.flush(); + zos.closeEntry(); // Ensure the entry is closed before ending the try block + LOGGER.info("{} rows exported in {}.", count.get(), filename); - zos.closeEntry(); } catch (Exception e) { LOGGER.error("Error during processing of {}: {}", filename, e.getMessage()); } From 6fb73fb867369babaffcd00ad1bc573d12cf4aa4 Mon Sep 17 00:00:00 2001 From: scezen Date: Mon, 5 Aug 2024 11:17:40 +0200 Subject: [PATCH 13/52] Add GTIN and ISBN specific pages + Controller Update --- .../ui/pages/OpenDataController.java | 22 +++++++++++++++++-- .../ui/services/OpenDataService.java | 1 - .../resources/templates/opendata-gtin.html | 12 ++++++++++ .../resources/templates/opendata-isbn.html | 12 ++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 ui/src/main/resources/templates/opendata-gtin.html create mode 100644 ui/src/main/resources/templates/opendata-isbn.html diff --git a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java index 5d610017a..5e71061b8 100644 --- a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java +++ b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java @@ -68,8 +68,8 @@ public SitemapEntry getExposedUrls() { public List getMultipleExposedUrls() { return Arrays.asList( SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, DEFAULT_PATH, 0.3, ChangeFreq.YEARLY), - SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/gtin-open-data.zip", 0.3, ChangeFreq.YEARLY), - SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/isbn-open-data.zip", 0.3, ChangeFreq.YEARLY) + SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/gtin", 0.3, ChangeFreq.YEARLY), + SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/isbn", 0.3, ChangeFreq.YEARLY) ); } @@ -85,6 +85,24 @@ public ModelAndView opendata(final HttpServletRequest request) { return ret; } + @GetMapping(value = {DEFAULT_PATH + "/gtin"}) + public ModelAndView opendataGtin(final HttpServletRequest request) { + final ModelAndView ret = uiService.defaultModelAndView("opendata-gtin", request); + ret.addObject("lastUpdated", openDataService.gtinLastUpdate()); + ret.addObject("fileSize", openDataService.gtinFileSize()); + ret.addObject("page", "gtin data"); + return ret; + } + + @GetMapping(value = {DEFAULT_PATH + "/isbn"}) + public ModelAndView opendataIsbn(final HttpServletRequest request) { + final ModelAndView ret = uiService.defaultModelAndView("opendata-isbn", request); + ret.addObject("lastUpdated", openDataService.isbnLastUpdate()); + ret.addObject("fileSize", openDataService.isbnFileSize()); + ret.addObject("page", "isbn data"); + return ret; + } + @GetMapping(path = "/opendata/gtin-open-data.zip") public void downloadGtinData(final HttpServletResponse response) throws IOException { downloadData(response, "gtin-open-data.zip", uiConfig.gtinZipFile()); diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 90eb915a6..6d6b47b25 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -64,7 +64,6 @@ public class OpenDataService { public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig){ this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; - generateOpendata(); } /** diff --git a/ui/src/main/resources/templates/opendata-gtin.html b/ui/src/main/resources/templates/opendata-gtin.html new file mode 100644 index 000000000..ff9df8bfa --- /dev/null +++ b/ui/src/main/resources/templates/opendata-gtin.html @@ -0,0 +1,12 @@ + + + + GTIN Open Data + + +

GTIN Open Data

+

Last Updated:

+

File Size:

+Download GTIN Data + + diff --git a/ui/src/main/resources/templates/opendata-isbn.html b/ui/src/main/resources/templates/opendata-isbn.html new file mode 100644 index 000000000..76156d78b --- /dev/null +++ b/ui/src/main/resources/templates/opendata-isbn.html @@ -0,0 +1,12 @@ + + + + ISBN Open Data + + +

ISBN Open Data

+

Last Updated:

+

File Size:

+Download ISBN Data + + From 071dd823acd914446ac052d7927f51d0fd97e602 Mon Sep 17 00:00:00 2001 From: goulven Date: Tue, 6 Aug 2024 15:01:42 +0200 Subject: [PATCH 14/52] Notes @scezen --- .../ui/services/OpenDataService.java | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 6d6b47b25..36dfa9be3 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -35,15 +35,35 @@ * * @author Goulven.Furet * + * TODO : Methods comments + * + * + * TODO : Add the below attributes only for ISBN dataset + * + * CLASSIFICATION DECITRE 1 Loisirs et Jeux 1 +CLASSIFICATION DECITRE 2 Jeux et activités 1 +CLASSIFICATION DECITRE 3 Coloriage Gommettes Autocollants +EDITEUR +FORMAT + +NB DE PAGES +SOUSCATEGORIE Enfant-jeunesse 1 +SOUSCATEGORIE2 Sport-et-loisirs + * + * */ public class OpenDataService { + private static final Logger LOGGER = LoggerFactory.getLogger(OpenDataService.class); + // Allowed download speed in kb + // TODO : In config (OpenDataConfig) private static final int DOWNLOAD_SPEED_KB = 256; - public static final int CONCURRENT_DOWNLOADS = 4; - private static final Logger LOGGER = LoggerFactory.getLogger(OpenDataService.class); + + private static final String ISBN_DATASET_FILENAME = "open4goods-isbn-dataset.csv"; + private static final String GTIN_DATASET_FILENAME = "open4goods-gtin-dataset.csv"; // The headers private static final String[] header = { "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", @@ -55,12 +75,9 @@ public class OpenDataService { // The flag that indicates wether opendata export is running or not private AtomicBoolean exportRunning = new AtomicBoolean(false); - private AtomicInteger concurrentDownloads = new AtomicInteger(0); + private AtomicInteger concurrentDownloadsCounter = new AtomicInteger(0); - private static final String ISBN_DATASET_FILENAME = "open4goods-isbn-dataset.csv"; - private static final String GTIN_DATASET_FILENAME = "open4goods-gtin-dataset.csv"; - @Autowired public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig){ this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; @@ -79,10 +96,10 @@ public InputStream limitedRateStream() throws TechnicalException, FileNotFoundEx RateLimiter rateLimiter = RateLimiter.create(DOWNLOAD_SPEED_KB * FileUtils.ONE_KB); // TODO : in conf - if (concurrentDownloads.get() >= CONCURRENT_DOWNLOADS) { + if (concurrentDownloadsCounter.get() >= CONCURRENT_DOWNLOADS) { throw new TechnicalException("Too many requests "); } else { - concurrentDownloads.incrementAndGet(); + concurrentDownloadsCounter.incrementAndGet(); } try { @@ -92,12 +109,12 @@ public InputStream limitedRateStream() throws TechnicalException, FileNotFoundEx @Override public void close() throws IOException { super.close(); - concurrentDownloads.decrementAndGet(); + concurrentDownloadsCounter.decrementAndGet(); LOGGER.info("Ending opendata dataset download"); } }; } catch (IOException e) { - concurrentDownloads.decrementAndGet(); + concurrentDownloadsCounter.decrementAndGet(); throw e; } } @@ -107,7 +124,7 @@ public void close() throws IOException { * * TODO : Schedule in conf */ - @Scheduled(initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) + @Scheduled( initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) public void generateOpendata() { if (exportRunning.getAndSet(true)) { LOGGER.error("Opendata export is already running"); @@ -165,6 +182,7 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File writer.writeNext(header); AtomicLong count = new AtomicLong(); + // TODO : Remove before MEP aggregatedDataRepository.exportAll().limit(500).filter(e -> invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) ).forEach(e -> { @@ -172,6 +190,8 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File writer.writeNext(toEntry(e)); }); writer.flush(); + + zos.closeEntry(); // Ensure the entry is closed before ending the try block LOGGER.info("{} rows exported in {}.", count.get(), filename); @@ -232,7 +252,7 @@ private String[] toEntry(Product data) { * (user stop the download) */ public void decrementDownloadCounter() { - concurrentDownloads.decrementAndGet(); + concurrentDownloadsCounter.decrementAndGet(); } @Cacheable(key = "#root.method.name + 'Isbn'", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) From 462ca8c28e80d14080f56d9e2063e49509aa05d3 Mon Sep 17 00:00:00 2001 From: scezen Date: Wed, 7 Aug 2024 22:56:04 +0200 Subject: [PATCH 15/52] Typo --- ui/src/main/resources/templates/opendata.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/main/resources/templates/opendata.html b/ui/src/main/resources/templates/opendata.html index 27ef4d80d..0fdbfdd13 100644 --- a/ui/src/main/resources/templates/opendata.html +++ b/ui/src/main/resources/templates/opendata.html @@ -7,7 +7,7 @@ - Base de données ISBN en libre accès | Nudger + Base de données GTIN, EAN et ISBN en libre accès | Nudger From f64566f2d2b5e8b010671c7efc9cda3883a39074 Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 9 Aug 2024 13:53:34 +0200 Subject: [PATCH 16/52] Adding OpenDataConfig --- .../org/open4goods/ui/config/AppConfig.java | 10 +++-- .../open4goods/ui/config/OpenDataConfig.java | 44 +++++++++++++++++++ ui/src/main/resources/application.yml | 10 ++++- 3 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java diff --git a/ui/src/main/java/org/open4goods/ui/config/AppConfig.java b/ui/src/main/java/org/open4goods/ui/config/AppConfig.java index 1fe0ecb37..f507dd510 100644 --- a/ui/src/main/java/org/open4goods/ui/config/AppConfig.java +++ b/ui/src/main/java/org/open4goods/ui/config/AppConfig.java @@ -186,11 +186,13 @@ IcecatService icecatFeatureService(UiConfig properties, RemoteFileCachingService // TODO : xmlMapper not injected because corruct the springdoc used one. Should use a @Primary derivation return new IcecatService(new XmlMapper(), properties.getIcecatFeatureConfig(), fileCachingService, properties.getRemoteCachingFolder(), brandService, verticalConfigService); } - - + + @Bean - OpenDataService openDataService(@Autowired ProductRepository aggregatedDataRepository, @Autowired UiConfig props) { - return new OpenDataService(aggregatedDataRepository, props); + OpenDataService openDataService(@Autowired ProductRepository aggregatedDataRepository, + @Autowired UiConfig props, + @Autowired OpenDataConfig openDataConfig) { + return new OpenDataService(aggregatedDataRepository, props, openDataConfig); } diff --git a/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java b/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java new file mode 100644 index 000000000..9f9cd8304 --- /dev/null +++ b/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java @@ -0,0 +1,44 @@ +package org.open4goods.ui.config; + +import org.springframework.context.annotation.Configuration; + +@Configuration +public class OpenDataConfig { + + private int downloadSpeedKb; + private int concurrentDownloads; + private long initialDelay; + private long fixedDelay; + + public int getDownloadSpeedKb() { + return downloadSpeedKb; + } + + public void setDownloadSpeedKb(int downloadSpeedKb) { + this.downloadSpeedKb = downloadSpeedKb; + } + + public int getConcurrentDownloads() { + return concurrentDownloads; + } + + public void setConcurrentDownloads(int concurrentDownloads) { + this.concurrentDownloads = concurrentDownloads; + } + + public long getInitialDelay() { + return initialDelay; + } + + public void setInitialDelay(long initialDelay) { + this.initialDelay = initialDelay; + } + + public long getFixedDelay() { + return fixedDelay; + } + + public void setFixedDelay(long fixedDelay) { + this.fixedDelay = fixedDelay; + } +} diff --git a/ui/src/main/resources/application.yml b/ui/src/main/resources/application.yml index 436eb6ab7..c9632e46c 100644 --- a/ui/src/main/resources/application.yml +++ b/ui/src/main/resources/application.yml @@ -68,9 +68,15 @@ apiConfig: # log-responses: false -tagListUrl: https://open4good.github.io/open4goods/maven/taglist/taglist.xml +tagListUrl: https://open4good.github.io/open4goods/maven/taglist/taglist.xml + +openDataConfig: + downloadSpeedKb: 256 + concurrentDownloads: 4 + initialDelay: 3600000 # 1 heure + fixedDelay: 604800000 # 1 semaine + - feedbackConfig: githubConfig: accessToken: GITHUB_ACCESS_TOKEN From 97274a1843a1c65bd449b08239ff6d6b1772e8e5 Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 9 Aug 2024 13:55:17 +0200 Subject: [PATCH 17/52] Frontend Tweaks + ISBN Headers --- .../ui/pages/OpenDataController.java | 2 + .../ui/services/OpenDataService.java | 97 ++++++--- .../resources/templates/opendata-gtin.html | 201 +++++++++++++++++- ui/src/main/resources/templates/opendata.html | 2 +- 4 files changed, 263 insertions(+), 39 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java index 5e71061b8..beb840960 100644 --- a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java +++ b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java @@ -89,6 +89,7 @@ public ModelAndView opendata(final HttpServletRequest request) { public ModelAndView opendataGtin(final HttpServletRequest request) { final ModelAndView ret = uiService.defaultModelAndView("opendata-gtin", request); ret.addObject("lastUpdated", openDataService.gtinLastUpdate()); + ret.addObject("countGTIN", openDataService.totalItemsGTIN()); ret.addObject("fileSize", openDataService.gtinFileSize()); ret.addObject("page", "gtin data"); return ret; @@ -98,6 +99,7 @@ public ModelAndView opendataGtin(final HttpServletRequest request) { public ModelAndView opendataIsbn(final HttpServletRequest request) { final ModelAndView ret = uiService.defaultModelAndView("opendata-isbn", request); ret.addObject("lastUpdated", openDataService.isbnLastUpdate()); + ret.addObject("countISBN", openDataService.totalItemsISBN()); ret.addObject("fileSize", openDataService.isbnFileSize()); ret.addObject("page", "isbn data"); return ret; diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 36dfa9be3..4f9ea302d 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -20,6 +20,7 @@ import org.open4goods.model.BarcodeType; import org.open4goods.model.constants.CacheConstants; import org.open4goods.model.product.Product; +import org.open4goods.ui.config.OpenDataConfig; import org.open4goods.ui.config.yml.UiConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,21 +57,27 @@ public class OpenDataService { private static final Logger LOGGER = LoggerFactory.getLogger(OpenDataService.class); - // Allowed download speed in kb - // TODO : In config (OpenDataConfig) - private static final int DOWNLOAD_SPEED_KB = 256; - public static final int CONCURRENT_DOWNLOADS = 4; +// // Allowed download speed in kb +// // TODO : In config (OpenDataConfig) +// private static final int DOWNLOAD_SPEED_KB = 256; +// public static final int CONCURRENT_DOWNLOADS = 4; private static final String ISBN_DATASET_FILENAME = "open4goods-isbn-dataset.csv"; private static final String GTIN_DATASET_FILENAME = "open4goods-gtin-dataset.csv"; - // The headers - private static final String[] header = { "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", + // GTIN headers + private static final String[] gtinHeader = { "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", "offers_count", "min_price", "min_price_compensation", "currency", "categories", "url" }; + // ISBN headers + private static final String[] isbnHeader = { "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", + "offers_count", "min_price", "min_price_compensation", "currency", "categories", "url", "editeur", "format" }; + + private ProductRepository aggregatedDataRepository; private UiConfig uiConfig; + private final OpenDataConfig openDataConfig; // The flag that indicates wether opendata export is running or not private AtomicBoolean exportRunning = new AtomicBoolean(false); @@ -78,9 +85,11 @@ public class OpenDataService { private AtomicInteger concurrentDownloadsCounter = new AtomicInteger(0); - public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig){ + public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig, OpenDataConfig openDataConfig){ this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; + this.openDataConfig = openDataConfig; + generateOpendata(); } /** @@ -93,10 +102,10 @@ public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiCo public InputStream limitedRateStream() throws TechnicalException, FileNotFoundException { // TODO : in conf - RateLimiter rateLimiter = RateLimiter.create(DOWNLOAD_SPEED_KB * FileUtils.ONE_KB); + RateLimiter rateLimiter = RateLimiter.create(openDataConfig.getDownloadSpeedKb() * FileUtils.ONE_KB); // TODO : in conf - if (concurrentDownloadsCounter.get() >= CONCURRENT_DOWNLOADS) { + if (concurrentDownloadsCounter.get() >= openDataConfig.getConcurrentDownloads()) { throw new TechnicalException("Too many requests "); } else { concurrentDownloadsCounter.incrementAndGet(); @@ -124,7 +133,7 @@ public void close() throws IOException { * * TODO : Schedule in conf */ - @Scheduled( initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) + //@Scheduled( initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) public void generateOpendata() { if (exportRunning.getAndSet(true)) { LOGGER.error("Opendata export is already running"); @@ -179,20 +188,23 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File ZipEntry entry = new ZipEntry(filename); zos.putNextEntry(entry); - writer.writeNext(header); + + // Correct header if GTIN or ISBN + if (barcodeType == BarcodeType.ISBN_13) { + writer.writeNext(isbnHeader); + } else { + writer.writeNext(gtinHeader); + } AtomicLong count = new AtomicLong(); - // TODO : Remove before MEP aggregatedDataRepository.exportAll().limit(500).filter(e -> invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) ).forEach(e -> { count.incrementAndGet(); - writer.writeNext(toEntry(e)); + writer.writeNext(toEntry(e, barcodeType == BarcodeType.ISBN_13)); }); writer.flush(); - - - zos.closeEntry(); // Ensure the entry is closed before ending the try block + zos.closeEntry(); // entry is closed before ending the try block LOGGER.info("{} rows exported in {}.", count.get(), filename); @@ -201,35 +213,40 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File } } + /** * Convert an aggregateddata pageSize a csv row * * @param data * @return */ - private String[] toEntry(Product data) { + private String[] toEntry(Product data, boolean isIsbn) { + String[] line; - String[] line = new String[header.length]; + // Adapter la taille du tableau au nombre de headers + if (isIsbn) { + line = new String[isbnHeader.length]; + } else { + line = new String[gtinHeader.length]; + } - // "gtin" + // "gtin" line[0] = data.gtin(); - // "brand" + // "brand" line[1] = data.brand(); - // "model" + // "model" line[2] = data.model(); - // "shortest_name" + // "shortest_name" line[3] = data.getNames().shortestOfferName(); - // "last_updated" + // "last_updated" line[4] = String.valueOf(data.getLastChange()); - // "gs1_country" + // "gs1_country" line[5] = data.getGtinInfos().getCountry(); - // "upcType" + // "upcType" line[6] = data.getGtinInfos().getUpcType().toString(); - - - // "offers_count" + // "offers_count" line[7] = String.valueOf(data.getOffersCount()); - // "min_price" + // "min_price" if (null != data.bestPrice()) { line[8] = String.valueOf(data.bestPrice().getPrice()); // "compensation" @@ -237,16 +254,21 @@ private String[] toEntry(Product data) { // "currency" line[10] = data.bestPrice().getCurrency().toString(); // "url" - // TODO(gof) : i18n the url line[12] = ""; //uiConfig.getBaseUrl(Locale.FRANCE) + data.getNames().getName(); } - - // Categories + // "Categories" line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); + // Modifier classe Product +// if (isIsbn) { +// line[13] = data.getEditeur(); +// line[14] = data.getFormat(); +// } + return line; } + /** * Used pageSize decrement the download counter, for instance when IOexception occurs * (user stop the download) @@ -275,6 +297,17 @@ public Date gtinLastUpdate() { return Date.from(Instant.ofEpochMilli(uiConfig.gtinZipFile().lastModified())); } + @Cacheable(key = "#root.method.name + 'Isbn'", cacheNames = CacheConstants.ONE_DAY_LOCAL_CACHE_NAME) + public long totalItemsISBN() { + return aggregatedDataRepository.countItemsByBarcodeType(BarcodeType.ISBN_13); + } + + @Cacheable(key = "#root.method.name + 'Gtin'", cacheNames = CacheConstants.ONE_DAY_LOCAL_CACHE_NAME) + public long totalItemsGTIN() { + return aggregatedDataRepository.countItemsByBarcodeType( + BarcodeType.GTIN_8, BarcodeType.GTIN_12, BarcodeType.GTIN_13, BarcodeType.GTIN_14); + } + /** * * @return number of items diff --git a/ui/src/main/resources/templates/opendata-gtin.html b/ui/src/main/resources/templates/opendata-gtin.html index ff9df8bfa..a1ee654a4 100644 --- a/ui/src/main/resources/templates/opendata-gtin.html +++ b/ui/src/main/resources/templates/opendata-gtin.html @@ -1,12 +1,201 @@ - + + - GTIN Open Data + + Base de données GTIN en libre accès | Nudger + + + + + + + + + + + + -

GTIN Open Data

-

Last Updated:

-

File Size:

-Download GTIN Data + + +
+
+
+
+
+

Open Data : Base de données GTIN en libre accès

+
+
+
+
+
+
+
+
+
+
+
+ code barre en opendata +
+
+

Accédez à notre base de données GTIN. Ces données sont mises à jour une fois par semaine et proviennent de notre agrégation de contenu autour des catalogues d'affiliation.

+
+
+

Qu'est-ce qu'un GTIN ?

+

+ Le GTIN (Global Trade Item Number) est un identifiant unique utilisé mondialement pour identifier les produits et services. Il est crucial pour le commerce international et facilite la gestion des stocks, la logistique, et les transactions commerciales. +

+
    +
  •    Utilisé internationalement
    • +
  •    Facilite la gestion des stocks
    • +
  •    Optimise la logistique
    • +
+

L'importance des GTIN pour l'écologie

+

+ Les GTIN jouent un rôle clé dans la traçabilité des produits, ce qui est essentiel pour une gestion écologique efficace. En permettant un suivi précis des produits tout au long de la chaîne d'approvisionnement, les GTIN aident à réduire le gaspillage et à optimiser l'utilisation des ressources. +

+
    +
  •    Réduction du gaspillage
    • +
  •    Optimisation des ressources
    • +
  •    Support à l'économie circulaire
    • +
+

Conditions d'utilisation

+
    +
  • Ces données sont délivrées sous licence ouverte, et sont donc librement utilisables, sans restrictions d'usages.
    • +
  • Vous devez indiquer la provenance des données, en mentionnant la page produit quand disponible, ou en ajoutant un lien vers la page d'accueil de nudger.fr.
    • +
+
+
+
+
+

Obtenir les données

+ + + + + + + + + + + + + + + +
Mise à jour
Taille du fichier
Nombre de produits
+
+
+ + + + + Télécharger + +
+
+
+
+ +

Format des données

+ +
+
+ Colonnes +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
barcodeLe code barre GTIN du produit
brandMarque du produit
modelRéférence constructeur du produit
last_updatedDernière mise à jour de ce produit, au format epoch en millisecondes
gs1_countryPays de l'organisme GS1 qui a délivré le code barre pour ce produit
gtinTypeType du code barre. Peut être GTIN_13, GTIN_8, GTIN_12, GTIN_14
offers_countNombre d'offres commerciales pour ce produit
min_pricePrix le plus faible pour ce produit
min_price_compensationCompensation carbone reversée, correspondant au prix le plus faible pour ce produit
currencyDevise du prix le plus faible et de la compensation carbone associée
categoriesCatégories dans lesquelles ce produit a été rencontré
urlURL de la fiche produit, uniquement si des offres commerciales sont disponibles
+
+
+
+
+
+
+
+ + + + + + + + + + + diff --git a/ui/src/main/resources/templates/opendata.html b/ui/src/main/resources/templates/opendata.html index 0fdbfdd13..41d980ca7 100644 --- a/ui/src/main/resources/templates/opendata.html +++ b/ui/src/main/resources/templates/opendata.html @@ -78,7 +78,7 @@

Obtenir les données

- + From 8b2b95ea93ab8d5a124864b4c8cd619ff1915397 Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 9 Aug 2024 14:35:43 +0200 Subject: [PATCH 18/52] Fix header logic and filtering for ISBN and GTIN datasets --- .../ui/services/OpenDataService.java | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 4f9ea302d..e012a8a0b 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -181,7 +181,7 @@ private void moveFile(File src, File dest) throws IOException { FileUtils.moveFile(src, dest); } - private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile, boolean invertCondition) throws IOException { + private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile, boolean isGtinFile) throws IOException { try (FileOutputStream fos = new FileOutputStream(zipFile); ZipOutputStream zos = new ZipOutputStream(fos); CSVWriter writer = new CSVWriter(new OutputStreamWriter(zos))) { @@ -189,22 +189,27 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File ZipEntry entry = new ZipEntry(filename); zos.putNextEntry(entry); - // Correct header if GTIN or ISBN - if (barcodeType == BarcodeType.ISBN_13) { + // Sélection des en-têtes basés sur le type de fichier + if (!isGtinFile) { // Fichier ISBN writer.writeNext(isbnHeader); - } else { + } else { // Fichier GTIN writer.writeNext(gtinHeader); } AtomicLong count = new AtomicLong(); - aggregatedDataRepository.exportAll().limit(500).filter(e -> - invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) - ).forEach(e -> { + aggregatedDataRepository.exportAll().limit(1000).filter(e -> { + if (isGtinFile) { + return e.getGtinInfos().getUpcType() != BarcodeType.ISBN_13; + } else { + return e.getGtinInfos().getUpcType() == BarcodeType.ISBN_13; + } + }).forEach(e -> { count.incrementAndGet(); - writer.writeNext(toEntry(e, barcodeType == BarcodeType.ISBN_13)); + writer.writeNext(toEntry(e, !isGtinFile)); }); + writer.flush(); - zos.closeEntry(); // entry is closed before ending the try block + zos.closeEntry(); LOGGER.info("{} rows exported in {}.", count.get(), filename); @@ -214,6 +219,7 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File } + /** * Convert an aggregateddata pageSize a csv row * @@ -260,10 +266,10 @@ private String[] toEntry(Product data, boolean isIsbn) { line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); // Modifier classe Product -// if (isIsbn) { -// line[13] = data.getEditeur(); -// line[14] = data.getFormat(); -// } + if (isIsbn) { + line[13] = data.getAttributes().getReferentielAttributes().get("editeur"); + line[14] = data.getAttributes().getReferentielAttributes().get("format"); + } return line; } From 4afd4a07ec44566c099b7455dc7095fcbeb4689c Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 9 Aug 2024 14:37:21 +0200 Subject: [PATCH 19/52] Typo --- .../open4goods/ui/services/OpenDataService.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index e012a8a0b..a082b981c 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -189,10 +189,10 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File ZipEntry entry = new ZipEntry(filename); zos.putNextEntry(entry); - // Sélection des en-têtes basés sur le type de fichier - if (!isGtinFile) { // Fichier ISBN + // Headers selon type de fichier + if (!isGtinFile) { writer.writeNext(isbnHeader); - } else { // Fichier GTIN + } else { writer.writeNext(gtinHeader); } @@ -265,11 +265,11 @@ private String[] toEntry(Product data, boolean isIsbn) { // "Categories" line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); - // Modifier classe Product - if (isIsbn) { - line[13] = data.getAttributes().getReferentielAttributes().get("editeur"); - line[14] = data.getAttributes().getReferentielAttributes().get("format"); - } +// // Modifier classe Product +// if (isIsbn) { +// line[13] = data.getAttributes().getReferentielAttributes().get("editeur"); +// line[14] = data.getAttributes().getReferentielAttributes().get("format"); +// } return line; } From 33cd7033de2e3782feda3fc84a5ff3fd9cf4bb4d Mon Sep 17 00:00:00 2001 From: goulven Date: Fri, 9 Aug 2024 15:17:52 +0200 Subject: [PATCH 20/52] Removing scheduled props --- .../open4goods/ui/config/OpenDataConfig.java | 18 +----------------- ui/src/main/resources/application.yml | 2 -- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java b/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java index 9f9cd8304..e9af258f6 100644 --- a/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java +++ b/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java @@ -7,8 +7,6 @@ public class OpenDataConfig { private int downloadSpeedKb; private int concurrentDownloads; - private long initialDelay; - private long fixedDelay; public int getDownloadSpeedKb() { return downloadSpeedKb; @@ -26,19 +24,5 @@ public void setConcurrentDownloads(int concurrentDownloads) { this.concurrentDownloads = concurrentDownloads; } - public long getInitialDelay() { - return initialDelay; - } - - public void setInitialDelay(long initialDelay) { - this.initialDelay = initialDelay; - } - - public long getFixedDelay() { - return fixedDelay; - } - - public void setFixedDelay(long fixedDelay) { - this.fixedDelay = fixedDelay; - } + } diff --git a/ui/src/main/resources/application.yml b/ui/src/main/resources/application.yml index c9632e46c..838475132 100644 --- a/ui/src/main/resources/application.yml +++ b/ui/src/main/resources/application.yml @@ -73,8 +73,6 @@ tagListUrl: https://open4good.github.io/open4goods/maven/taglist/taglist.xml openDataConfig: downloadSpeedKb: 256 concurrentDownloads: 4 - initialDelay: 3600000 # 1 heure - fixedDelay: 604800000 # 1 semaine feedbackConfig: From 6a119b90c10e1633bc30aed12bee2dbb91bfcfcd Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 9 Aug 2024 15:20:15 +0200 Subject: [PATCH 21/52] ProductRepository --- .../main/java/org/open4goods/dao/ProductRepository.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/commons/src/main/java/org/open4goods/dao/ProductRepository.java b/commons/src/main/java/org/open4goods/dao/ProductRepository.java index 8effaa784..d85ea9ae2 100644 --- a/commons/src/main/java/org/open4goods/dao/ProductRepository.java +++ b/commons/src/main/java/org/open4goods/dao/ProductRepository.java @@ -14,6 +14,7 @@ import org.open4goods.config.yml.ui.VerticalConfig; import org.open4goods.exceptions.ResourceNotFoundException; +import org.open4goods.model.BarcodeType; import org.open4goods.model.constants.CacheConstants; import org.open4goods.model.product.Product; import org.open4goods.store.repository.ProductIndexationWorker; @@ -497,6 +498,12 @@ public Long countMainIndex() { return elasticsearchTemplate.count(Query.findAll(), current_index); } + @Cacheable(cacheNames = CacheConstants.ONE_DAY_LOCAL_CACHE_NAME) + public long countItemsByBarcodeType(BarcodeType... barcodeTypes) { + Criteria criteria = new Criteria("gtinInfos.upcType").in((Object[]) barcodeTypes); + CriteriaQuery query = new CriteriaQuery(criteria); + return elasticsearchTemplate.count(query, current_index); + } @Cacheable(cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) public Long countMainIndexHavingPrice() { CriteriaQuery query = new CriteriaQuery(getValidDateQuery()); From 4610152e1ea582451afbe2efc0dd3df327118b80 Mon Sep 17 00:00:00 2001 From: goulven Date: Fri, 9 Aug 2024 15:54:11 +0200 Subject: [PATCH 22/52] Working session --- .../org/open4goods/dao/ProductRepository.java | 15 +++ .../org/open4goods/model/BarcodeType.java | 7 +- .../model/product/AggregatedAttributes.java | 1 + .../ui/services/OpenDataService.java | 113 +++++++++++++++--- 4 files changed, 116 insertions(+), 20 deletions(-) diff --git a/commons/src/main/java/org/open4goods/dao/ProductRepository.java b/commons/src/main/java/org/open4goods/dao/ProductRepository.java index d85ea9ae2..bc864b235 100644 --- a/commons/src/main/java/org/open4goods/dao/ProductRepository.java +++ b/commons/src/main/java/org/open4goods/dao/ProductRepository.java @@ -126,6 +126,21 @@ public Stream exportAll() { .map(SearchHit::getContent); } + /** + * Export all aggregated data, corresponding to the given Barcodes + * + * @return + */ + public Stream exportAll(BarcodeType... barcodeTypes) { + + Criteria criteria = new Criteria("gtinInfos.upcType").in((Object[]) barcodeTypes); + CriteriaQuery query = new CriteriaQuery(criteria); + + return elasticsearchTemplate.searchForStream(query, Product.class, current_index).stream() + .map(SearchHit::getContent); + } + + public Stream searchInValidPrices(String query, final String indexName, int from, int to) { Criteria c = new Criteria().expression(query).and(getValidDateQuery()); diff --git a/commons/src/main/java/org/open4goods/model/BarcodeType.java b/commons/src/main/java/org/open4goods/model/BarcodeType.java index 29fee7da0..ef562e862 100644 --- a/commons/src/main/java/org/open4goods/model/BarcodeType.java +++ b/commons/src/main/java/org/open4goods/model/BarcodeType.java @@ -1,7 +1,12 @@ package org.open4goods.model; public enum BarcodeType { - ISBN_13, GTIN_13, GTIN_8, GTIN_12, GTIN_14, UNKNOWN, + ISBN_13, + GTIN_13, + GTIN_8, + GTIN_12, + GTIN_14, + UNKNOWN, } diff --git a/commons/src/main/java/org/open4goods/model/product/AggregatedAttributes.java b/commons/src/main/java/org/open4goods/model/product/AggregatedAttributes.java index dd42219e5..4f7c424c5 100644 --- a/commons/src/main/java/org/open4goods/model/product/AggregatedAttributes.java +++ b/commons/src/main/java/org/open4goods/model/product/AggregatedAttributes.java @@ -26,6 +26,7 @@ public class AggregatedAttributes { //TODO: rename + // TODO : specifiy indexation rules private Map aggregatedAttributes = new HashMap<>(); diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index a082b981c..16a9188b8 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -189,23 +189,25 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File ZipEntry entry = new ZipEntry(filename); zos.putNextEntry(entry); + + BarcodeType[] types = null; + // Headers selon type de fichier - if (!isGtinFile) { - writer.writeNext(isbnHeader); - } else { + if (isGtinFile) { writer.writeNext(gtinHeader); + types = new BarcodeType[] { BarcodeType.GTIN_8, BarcodeType.GTIN_12, BarcodeType.GTIN_13, BarcodeType.GTIN_14 }; + } else { + writer.writeNext(isbnHeader); + types = new BarcodeType[] { BarcodeType.ISBN_13 }; } AtomicLong count = new AtomicLong(); - aggregatedDataRepository.exportAll().limit(1000).filter(e -> { - if (isGtinFile) { - return e.getGtinInfos().getUpcType() != BarcodeType.ISBN_13; - } else { - return e.getGtinInfos().getUpcType() == BarcodeType.ISBN_13; - } - }).forEach(e -> { - count.incrementAndGet(); - writer.writeNext(toEntry(e, !isGtinFile)); + //TODO : Remind to remove limit + aggregatedDataRepository.exportAll(types) + .limit(1000) + .forEach(e -> { + count.incrementAndGet(); + writer.writeNext(isGtinFile ? toGtinEntry(e) : toIsbnEntry(e)); }); writer.flush(); @@ -226,15 +228,57 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File * @param data * @return */ - private String[] toEntry(Product data, boolean isIsbn) { - String[] line; + private String[] toGtinEntry(Product data) { + String[] line = new String[gtinHeader.length]; - // Adapter la taille du tableau au nombre de headers - if (isIsbn) { - line = new String[isbnHeader.length]; - } else { - line = new String[gtinHeader.length]; + // "gtin" + line[0] = data.gtin(); + // "brand" + line[1] = data.brand(); + // "model" + line[2] = data.model(); + // "shortest_name" + line[3] = data.getNames().shortestOfferName(); + // "last_updated" + line[4] = String.valueOf(data.getLastChange()); + // "gs1_country" + line[5] = data.getGtinInfos().getCountry(); + // "upcType" + line[6] = data.getGtinInfos().getUpcType().toString(); + // "offers_count" + line[7] = String.valueOf(data.getOffersCount()); + // "min_price" + if (null != data.bestPrice()) { + line[8] = String.valueOf(data.bestPrice().getPrice()); + // "compensation" + line[9] = String.valueOf(data.bestPrice().getCompensation()); + // "currency" + line[10] = data.bestPrice().getCurrency().toString(); + // "url" + line[12] = ""; //uiConfig.getBaseUrl(Locale.FRANCE) + data.getNames().getName(); } + // "Categories" + line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); + +// // Modifier classe Product +// if (isIsbn) { +// line[13] = data.getAttributes().getReferentielAttributes().get("editeur"); +// line[14] = data.getAttributes().getReferentielAttributes().get("format"); +// } + + return line; + } + + + + /** + * Convert an aggregateddata pageSize a csv row + * + * @param data + * @return + */ + private String[] toIsbnEntry(Product data) { + String[] line = new String[isbnHeader.length]; // "gtin" line[0] = data.gtin(); @@ -264,6 +308,10 @@ private String[] toEntry(Product data, boolean isIsbn) { } // "Categories" line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); + + + + String attrVal = getAttribute(data, "editeur"); // // Modifier classe Product // if (isIsbn) { @@ -274,7 +322,34 @@ private String[] toEntry(Product data, boolean isIsbn) { return line; } + /** + * Try a direct access to aggregatedattributes, if fail iterate over unmapped ones + * TODO : Review when all attributes acessible by map + * @param data + * @param key + * @return + */ + private String getAttribute(Product data, String key) { + + // Direct check against aggregatedAttributes + String value = data.getAttributes().getAggregatedAttributes().get(key).getValue(); + + if (StringUtils.isEmpty(value)) { + // Checking in unmapped attributes + + value = data.getAttributes().getUnmapedAttributes().stream() + .filter(a -> a.getName().equalsIgnoreCase(key)) + .findFirst() + .map(a -> a.getValue()) + .orElse(null); + + } + return value; + } + + + /** * Used pageSize decrement the download counter, for instance when IOexception occurs * (user stop the download) From b13df5367e4e060a86afe9c473b4dbba0daf415f Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 9 Aug 2024 21:06:50 +0200 Subject: [PATCH 23/52] Specific ISBN attributes and null values handling + Refactor and cleanup --- .../ui/pages/OpenDataController.java | 12 - .../ui/services/OpenDataService.java | 299 ++++++++---------- 2 files changed, 129 insertions(+), 182 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java index beb840960..ffa4ed8e1 100644 --- a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java +++ b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java @@ -35,30 +35,18 @@ public class OpenDataController implements SitemapExposedController{ public static final String DEFAULT_PATH="/opendata"; - private static final Logger LOGGER = LoggerFactory.getLogger(OpenDataController.class); - // The siteConfig private final OpenDataService openDataService; private @Autowired UiService uiService; private final UiConfig uiConfig; - @Autowired public OpenDataController(OpenDataService openDataService, UiConfig uiConfig) { this.openDataService = openDataService; this.uiConfig = uiConfig; } - /** - * The Home page. - * - * @param request - * @param response - * @return - * @throws UnirestException - */ - @Override public SitemapEntry getExposedUrls() { return SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, DEFAULT_PATH, 0.3, ChangeFreq.YEARLY); diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 16a9188b8..79e500aa4 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -5,6 +5,7 @@ import java.text.StringCharacterIterator; import java.time.Instant; import java.util.Date; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -19,6 +20,7 @@ import org.open4goods.helper.ThrottlingInputStream; import org.open4goods.model.BarcodeType; import org.open4goods.model.constants.CacheConstants; +import org.open4goods.model.product.AggregatedAttribute; import org.open4goods.model.product.Product; import org.open4goods.ui.config.OpenDataConfig; import org.open4goods.ui.config.yml.UiConfig; @@ -32,48 +34,29 @@ import com.opencsv.CSVWriter; /** - * Service in charge of generating the CSV opendata file - * - * @author Goulven.Furet - * - * TODO : Methods comments - * - * - * TODO : Add the below attributes only for ISBN dataset - * - * CLASSIFICATION DECITRE 1 Loisirs et Jeux 1 -CLASSIFICATION DECITRE 2 Jeux et activités 1 -CLASSIFICATION DECITRE 3 Coloriage Gommettes Autocollants -EDITEUR -FORMAT - -NB DE PAGES -SOUSCATEGORIE Enfant-jeunesse 1 -SOUSCATEGORIE2 Sport-et-loisirs - * - * + * Service responsible for generating and managing the CSV opendata files. + * Handles the creation of GTIN and ISBN datasets, their compression into ZIP files, + * and the management of download limits and rates. */ public class OpenDataService { private static final Logger LOGGER = LoggerFactory.getLogger(OpenDataService.class); -// // Allowed download speed in kb -// // TODO : In config (OpenDataConfig) -// private static final int DOWNLOAD_SPEED_KB = 256; -// public static final int CONCURRENT_DOWNLOADS = 4; - - private static final String ISBN_DATASET_FILENAME = "open4goods-isbn-dataset.csv"; private static final String GTIN_DATASET_FILENAME = "open4goods-gtin-dataset.csv"; - // GTIN headers - private static final String[] gtinHeader = { "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", - "offers_count", "min_price", "min_price_compensation", "currency", "categories", "url" }; - - // ISBN headers - private static final String[] isbnHeader = { "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", - "offers_count", "min_price", "min_price_compensation", "currency", "categories", "url", "editeur", "format" }; + private static final String[] GTIN_HEADER = { + "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", + "offers_count", "min_price", "min_price_compensation", "currency", "categories", "url" + }; + private static final String[] ISBN_HEADER = { + "code", "brand", "model", "name", "last_updated", "gs1_country", "gtintype", + "offers_count", "min_price", "min_price_compensation", "currency", "categories", "url", + "editeur", "format", "nb de pages", + "classification decitre 1", "classification decitre 2", "classification decitre 3", + "souscategorie", "souscategorie2" + }; private ProductRepository aggregatedDataRepository; private UiConfig uiConfig; @@ -81,7 +64,6 @@ public class OpenDataService { // The flag that indicates wether opendata export is running or not private AtomicBoolean exportRunning = new AtomicBoolean(false); - private AtomicInteger concurrentDownloadsCounter = new AtomicInteger(0); @@ -93,18 +75,13 @@ public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiCo } /** - * - * @return a limited stream pageSize the opendata set. Limited in bandwith, and - * limited in number of conccurent downloads Limited - * @throws FileNotFoundException - * @throws TechnicalException + * Provides a limited bandwidth stream for downloading the opendata file. + * Enforces a limit on the number of concurrent downloads. */ public InputStream limitedRateStream() throws TechnicalException, FileNotFoundException { - // TODO : in conf RateLimiter rateLimiter = RateLimiter.create(openDataConfig.getDownloadSpeedKb() * FileUtils.ONE_KB); - // TODO : in conf if (concurrentDownloadsCounter.get() >= openDataConfig.getConcurrentDownloads()) { throw new TechnicalException("Too many requests "); } else { @@ -113,8 +90,7 @@ public InputStream limitedRateStream() throws TechnicalException, FileNotFoundEx try { LOGGER.info("Starting opendata dataset download"); - return new ThrottlingInputStream(new BufferedInputStream(new FileInputStream(uiConfig.openDataFile())), - rateLimiter) { + return new ThrottlingInputStream(new BufferedInputStream(new FileInputStream(uiConfig.openDataFile())), rateLimiter) { @Override public void close() throws IOException { super.close(); @@ -129,12 +105,13 @@ public void close() throws IOException { } /** - * Iterates over all aggregated data to generate the zipped opendata CSV file. - * + * Generates the opendata CSV files and compresses them into ZIP files. + * This method is scheduled to run periodically. * TODO : Schedule in conf */ - //@Scheduled( initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) + @Scheduled(initialDelay = 1000L * 3600, fixedDelay = 1000L * 3600 * 24 * 7) public void generateOpendata() { + if (exportRunning.getAndSet(true)) { LOGGER.error("Opendata export is already running"); return; @@ -152,15 +129,17 @@ public void generateOpendata() { } } - private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile) throws IOException { - processAndCreateZip(filename, barcodeType, zipFile, false); - } - + /** + * Prepares the necessary directories for storing temporary files. + */ private void prepareDirectories() throws IOException { uiConfig.tmpIsbnZipFile().getParentFile().mkdirs(); uiConfig.tmpGtinZipFile().getParentFile().mkdirs(); } + /** + * Processes and creates the ZIP files for the opendata. + */ private void processDataFiles() throws IOException { LOGGER.info("Starting process for ISBN_13"); processAndCreateZip(ISBN_DATASET_FILENAME, BarcodeType.ISBN_13, uiConfig.tmpIsbnZipFile()); @@ -169,11 +148,17 @@ private void processDataFiles() throws IOException { processAndCreateZip(GTIN_DATASET_FILENAME, BarcodeType.ISBN_13, uiConfig.tmpGtinZipFile(), true); } + /** + * Moves the temporary files to their final destination. + */ private void moveTmpFilesToFinalDestination() throws IOException { moveFile(uiConfig.tmpIsbnZipFile(), uiConfig.isbnZipFile()); moveFile(uiConfig.tmpGtinZipFile(), uiConfig.gtinZipFile()); } + /** + * Moves a file from the source to the destination. + */ private void moveFile(File src, File dest) throws IOException { if (dest.exists()) { FileUtils.deleteQuietly(dest); @@ -181,6 +166,19 @@ private void moveFile(File src, File dest) throws IOException { FileUtils.moveFile(src, dest); } + private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile) throws IOException { + processAndCreateZip(filename, barcodeType, zipFile, false); + } + + /** + * Processes the data and creates a ZIP file for the specified barcode type. + * + * @param filename The name of the file to be created. + * @param barcodeType The type of barcode being processed. + * @param zipFile The file object representing the ZIP file to be created. + * @param isGtinFile Indicates if the file is a GTIN file. + * @throws IOException If there is an error during file creation or writing. + */ private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile, boolean isGtinFile) throws IOException { try (FileOutputStream fos = new FileOutputStream(zipFile); ZipOutputStream zos = new ZipOutputStream(fos); @@ -189,26 +187,23 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File ZipEntry entry = new ZipEntry(filename); zos.putNextEntry(entry); - - BarcodeType[] types = null; - - // Headers selon type de fichier + // Set the appropriate header and barcode types + BarcodeType[] types; if (isGtinFile) { - writer.writeNext(gtinHeader); - types = new BarcodeType[] { BarcodeType.GTIN_8, BarcodeType.GTIN_12, BarcodeType.GTIN_13, BarcodeType.GTIN_14 }; + writer.writeNext(GTIN_HEADER); + types = new BarcodeType[]{BarcodeType.GTIN_8, BarcodeType.GTIN_12, BarcodeType.GTIN_13, BarcodeType.GTIN_14}; } else { - writer.writeNext(isbnHeader); - types = new BarcodeType[] { BarcodeType.ISBN_13 }; + writer.writeNext(ISBN_HEADER); + types = new BarcodeType[]{BarcodeType.ISBN_13}; } AtomicLong count = new AtomicLong(); - //TODO : Remind to remove limit + aggregatedDataRepository.exportAll(types) - .limit(1000) - .forEach(e -> { - count.incrementAndGet(); - writer.writeNext(isGtinFile ? toGtinEntry(e) : toIsbnEntry(e)); - }); + .forEach(e -> { + count.incrementAndGet(); + writer.writeNext(isGtinFile ? toGtinEntry(e) : toIsbnEntry(e)); + }); writer.flush(); zos.closeEntry(); @@ -223,136 +218,100 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File /** - * Convert an aggregateddata pageSize a csv row - * - * @param data - * @return + * Converts a Product object into a CSV row for GTIN. */ private String[] toGtinEntry(Product data) { - String[] line = new String[gtinHeader.length]; - - // "gtin" - line[0] = data.gtin(); - // "brand" - line[1] = data.brand(); - // "model" - line[2] = data.model(); - // "shortest_name" - line[3] = data.getNames().shortestOfferName(); - // "last_updated" - line[4] = String.valueOf(data.getLastChange()); - // "gs1_country" - line[5] = data.getGtinInfos().getCountry(); - // "upcType" - line[6] = data.getGtinInfos().getUpcType().toString(); - // "offers_count" - line[7] = String.valueOf(data.getOffersCount()); - // "min_price" + String[] line = new String[GTIN_HEADER.length]; + + line[0] = data.gtin(); // "code" + line[1] = data.brand(); // "brand" + line[2] = data.model(); // "model" + line[3] = data.getNames().shortestOfferName(); // "name" + line[4] = String.valueOf(data.getLastChange()); // "last_updated" + line[5] = data.getGtinInfos().getCountry(); // "gs1_country" + line[6] = data.getGtinInfos().getUpcType().toString(); // "gtinType" + line[7] = String.valueOf(data.getOffersCount()); // "offers_count" + if (null != data.bestPrice()) { - line[8] = String.valueOf(data.bestPrice().getPrice()); - // "compensation" - line[9] = String.valueOf(data.bestPrice().getCompensation()); - // "currency" - line[10] = data.bestPrice().getCurrency().toString(); - // "url" - line[12] = ""; //uiConfig.getBaseUrl(Locale.FRANCE) + data.getNames().getName(); + line[8] = String.valueOf(data.bestPrice().getPrice()); // "min_price" + line[9] = String.valueOf(data.bestPrice().getCompensation()); // "min_price_compensation" + line[10] = data.bestPrice().getCurrency().toString(); // "currency" + line[12] = ""; // TODO: Construct the URL } - // "Categories" - line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); -// // Modifier classe Product -// if (isIsbn) { -// line[13] = data.getAttributes().getReferentielAttributes().get("editeur"); -// line[14] = data.getAttributes().getReferentielAttributes().get("format"); -// } + line[11] = StringUtils.join(data.getDatasourceCategories(), " ; "); // "categories" return line; } - - /** - * Convert an aggregateddata pageSize a csv row - * - * @param data - * @return + * Converts a Product object into a CSV row for ISBN. */ private String[] toIsbnEntry(Product data) { - String[] line = new String[isbnHeader.length]; - - // "gtin" - line[0] = data.gtin(); - // "brand" - line[1] = data.brand(); - // "model" - line[2] = data.model(); - // "shortest_name" - line[3] = data.getNames().shortestOfferName(); - // "last_updated" - line[4] = String.valueOf(data.getLastChange()); - // "gs1_country" - line[5] = data.getGtinInfos().getCountry(); - // "upcType" - line[6] = data.getGtinInfos().getUpcType().toString(); - // "offers_count" - line[7] = String.valueOf(data.getOffersCount()); - // "min_price" + String[] line = new String[ISBN_HEADER.length]; + + line[0] = data.gtin(); // "code" + line[1] = data.brand(); // "brand" + line[2] = data.model(); // "model" + line[3] = data.getNames().shortestOfferName(); // "name" + line[4] = String.valueOf(data.getLastChange()); // "last_updated" + line[5] = data.getGtinInfos().getCountry(); // "gs1_country" + line[6] = data.getGtinInfos().getUpcType().toString(); // "gtintype" + line[7] = String.valueOf(data.getOffersCount()); // "offers_count" + if (null != data.bestPrice()) { - line[8] = String.valueOf(data.bestPrice().getPrice()); - // "compensation" - line[9] = String.valueOf(data.bestPrice().getCompensation()); - // "currency" - line[10] = data.bestPrice().getCurrency().toString(); - // "url" - line[12] = ""; //uiConfig.getBaseUrl(Locale.FRANCE) + data.getNames().getName(); + line[8] = String.valueOf(data.bestPrice().getPrice()); // "min_price" + line[9] = String.valueOf(data.bestPrice().getCompensation()); // "min_price_compensation" + line[10] = data.bestPrice().getCurrency().toString(); // "currency" + line[12] = ""; // TODO: Construct the URL } - // "Categories" - line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); - - - - String attrVal = getAttribute(data, "editeur"); - -// // Modifier classe Product -// if (isIsbn) { -// line[13] = data.getAttributes().getReferentielAttributes().get("editeur"); -// line[14] = data.getAttributes().getReferentielAttributes().get("format"); -// } + + line[11] = StringUtils.join(data.getDatasourceCategories(), " ; "); // "categories" + + line[13] = getAttribute(data, "EDITEUR"); + line[14] = getAttribute(data, "FORMAT"); + line[15] = getAttribute(data, "NB DE PAGES"); + line[16] = getAttribute(data, "CLASSIFICATION DECITRE 1"); + line[17] = getAttribute(data, "CLASSIFICATION DECITRE 2"); + line[18] = getAttribute(data, "CLASSIFICATION DECITRE 3"); + line[19] = getAttribute(data, "SOUSCATEGORIE"); + line[20] = getAttribute(data, "SOUSCATEGORIE2"); return line; } /** - * Try a direct access to aggregatedattributes, if fail iterate over unmapped ones + * Retrieves a specific attribute from the product data. * TODO : Review when all attributes acessible by map - * @param data - * @param key - * @return */ private String getAttribute(Product data, String key) { - - // Direct check against aggregatedAttributes - String value = data.getAttributes().getAggregatedAttributes().get(key).getValue(); - + + String value = null; + + Map aggregatedAttributes = data.getAttributes().getAggregatedAttributes(); + + if (aggregatedAttributes.containsKey(key)) { + AggregatedAttribute attribute = aggregatedAttributes.get(key); + + if (attribute != null) { + value = attribute.getValue(); + } + } + if (StringUtils.isEmpty(value)) { // Checking in unmapped attributes - value = data.getAttributes().getUnmapedAttributes().stream() - .filter(a -> a.getName().equalsIgnoreCase(key)) - .findFirst() - .map(a -> a.getValue()) - .orElse(null); - + .filter(a -> a.getName().equalsIgnoreCase(key)) + .findFirst() + .map(AggregatedAttribute::getValue) + .orElse(null); } return value; } - - - + /** - * Used pageSize decrement the download counter, for instance when IOexception occurs - * (user stop the download) + * Decrements the download counter, for instance when an IOException occurs (e.g., user stops the download). */ public void decrementDownloadCounter() { concurrentDownloadsCounter.decrementAndGet(); @@ -390,8 +349,9 @@ public long totalItemsGTIN() { } /** + * Retrieves the total number of items in the main index. * - * @return number of items + * @return The total number of items. */ @Cacheable(key = "#root.method.name", cacheNames = CacheConstants.ONE_DAY_LOCAL_CACHE_NAME) public long totalItems() { @@ -399,9 +359,10 @@ public long totalItems() { } /** - * Convert a size in human readable form - * @param bytes - * @return + * Converts a size in bytes to a human-readable format. + * + * @param bytes The size in bytes. + * @return The size in a human-readable format (e.g., "1.2 MB"). */ public static String humanReadableByteCountBin(long bytes) { if (-1000 < bytes && bytes < 1000) { @@ -414,6 +375,4 @@ public static String humanReadableByteCountBin(long bytes) { } return String.format("%.1f %cB", bytes / 1000.0, ci.current()); } - - } From e587e071053b558445cf526cced78362bbc1f651 Mon Sep 17 00:00:00 2001 From: scezen Date: Tue, 13 Aug 2024 09:49:08 +0200 Subject: [PATCH 24/52] Cleanup --- .../ui/pages/OpenDataController.java | 2 + .../ui/services/OpenDataService.java | 7 +- .../resources/templates/opendata-isbn.html | 183 +++++++++++++- ui/src/main/resources/templates/opendata.html | 227 +++++++----------- 4 files changed, 264 insertions(+), 155 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java index ffa4ed8e1..9d2bf8f7f 100644 --- a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java +++ b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java @@ -65,6 +65,8 @@ public List getMultipleExposedUrls() { public ModelAndView opendata(final HttpServletRequest request) { final ModelAndView ret = uiService.defaultModelAndView("opendata", request); ret.addObject("count", openDataService.totalItems()); + ret.addObject("countGTIN", openDataService.totalItemsGTIN()); + ret.addObject("countISBN", openDataService.totalItemsISBN()); ret.addObject("isbnLastUpdated", openDataService.isbnLastUpdate()); ret.addObject("isbnFileSize", openDataService.isbnFileSize()); ret.addObject("gtinLastUpdated", openDataService.gtinLastUpdate()); diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 79e500aa4..71f26e222 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -71,7 +71,6 @@ public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiCo this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; this.openDataConfig = openDataConfig; - generateOpendata(); } /** @@ -236,7 +235,7 @@ private String[] toGtinEntry(Product data) { line[8] = String.valueOf(data.bestPrice().getPrice()); // "min_price" line[9] = String.valueOf(data.bestPrice().getCompensation()); // "min_price_compensation" line[10] = data.bestPrice().getCurrency().toString(); // "currency" - line[12] = ""; // TODO: Construct the URL + line[12] = ""; // TODO: uiConfig.getBaseUrl(Locale.FRANCE) + data.getNames().getName(); } line[11] = StringUtils.join(data.getDatasourceCategories(), " ; "); // "categories" @@ -263,7 +262,7 @@ private String[] toIsbnEntry(Product data) { line[8] = String.valueOf(data.bestPrice().getPrice()); // "min_price" line[9] = String.valueOf(data.bestPrice().getCompensation()); // "min_price_compensation" line[10] = data.bestPrice().getCurrency().toString(); // "currency" - line[12] = ""; // TODO: Construct the URL + line[12] = ""; // TODO: uiConfig.getBaseUrl(Locale.FRANCE) + data.getNames().getName(); } line[11] = StringUtils.join(data.getDatasourceCategories(), " ; "); // "categories" @@ -282,7 +281,7 @@ private String[] toIsbnEntry(Product data) { /** * Retrieves a specific attribute from the product data. - * TODO : Review when all attributes acessible by map + * TODO : Review when all attributes accessible by map */ private String getAttribute(Product data, String key) { diff --git a/ui/src/main/resources/templates/opendata-isbn.html b/ui/src/main/resources/templates/opendata-isbn.html index 76156d78b..b997a2d7f 100644 --- a/ui/src/main/resources/templates/opendata-isbn.html +++ b/ui/src/main/resources/templates/opendata-isbn.html @@ -1,12 +1,183 @@ - + + - ISBN Open Data + + Base de données ISBN en libre accès | Nudger + + + + + + + + + + + + -

ISBN Open Data

-

Last Updated:

-

File Size:

-Download ISBN Data + + +
+
+
+
+
+

Open Data : Base de données ISBN en libre accès

+
+
+
+
+
+
+
+
+
+
+
+ code barre en opendata +
+
+

Libérez les données ! Accédez à notre base de données ISBN. Ces données sont mises à jour une fois par semaine et proviennent de notre agrégation de contenu autour des catalogues d'affiliation.

+
+
+

Conditions d'utilisation

+
    +
  • Ces données sont délivrées sous licence ouverte, et sont donc librement utilisables, sans restrictions d'usages.
    • +
  • Vous devez indiquer la provenance des données, en mentionnant la page produit quand disponible, ou en ajoutant un lien vers la page d'accueil de nudger.fr.
    • +
+
+
+
+
+

Obtenir les données

+
mise à jourMise à jour
+ + + + + + + + + + + + + + +
mise à jour
Taille du fichier
Nombre de produits
+ +
+ + + + + Télécharger + +
+ + + + +

Format des données

+ +
+
+ Colonnes +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
barcodeLe code barre du produit, qui est un ISBN
brandMarque de l'éditeur
modelRéférence de l'ouvrage
last_updatedDernière mise à jour de ce produit, au format epoch en millisecondes
gs1_countryPays de l'organisme GS1 qui a délivré le code barre pour ce produit
gtinTypeType du code barre. Pour ISBN, cela sera toujours ISBN_13
offers_countNombre d'offres commerciales pour ce produit
min_pricePrix le plus faible pour ce produit
min_price_compensationCompensation carbone reversée, correspondant au prix le plus faible pour ce produit
currencyDevise du prix le plus faible et de la compensation carbone associée
categoriesCatégories dans lesquelles ce produit a été rencontré
urlURL de la fiche produit, uniquement si des offres commerciales sont disponibles
+
+ + + + + + + + + + + + + + + + + diff --git a/ui/src/main/resources/templates/opendata.html b/ui/src/main/resources/templates/opendata.html index 41d980ca7..476d12d8f 100644 --- a/ui/src/main/resources/templates/opendata.html +++ b/ui/src/main/resources/templates/opendata.html @@ -46,161 +46,98 @@

Open Data : Base de données GTIN, EAN et ISBN en lib - - -
-
-
-
-
- -
-
- code barre en opendata -
-
-

Liberez, la donnée ! Nous mettons à disposition une extraction complète de notre base de données, celle-ci est notament utile pour le grand nombre d'articles identifiés par leurs codes barres (gtin / ean13 / UPC / ISBN). Ces données sont mises à jour une fois par semaine, et proviennent de l'aggrégation de contenu que nous réalisons autour des catalogues d'affiliations.

-
-
- -

Conditions d'utilisation

+ + +
+
+
+
+
+
+
+ code barre en opendata +
+
+

Libérez la donnée ! Nous mettons à disposition une extraction complète de notre base de données, celle-ci est notamment utile pour le grand nombre d'articles identifiés par leurs codes barres (GTIN / EAN13 / UPC / ISBN). Ces données sont mises à jour une fois par semaine, et proviennent de l'agrégation de contenu que nous réalisons autour des catalogues d'affiliations.

+
+
+ +

Conditions d'utilisation

  • Ces données sont délivrées sous licence ouverte, et sont donc librement utilisables, sans restrictions d'usages.
  • Vous devez par contre indiquer la provenance de ces données, en mentionnant la page produit quand elle est disponible, ou avec un lien vers la page d'accueil de nudger.fr quand la page produit n'est pas disponible.
    • -
- -
-
-
-
-

Obtenir les données

- - - - - - - - - - - - - - - - - -
Mise à jour
Taille du fichier
Nombre de produits
- + + +
+
+
+
+

 

+ + + + + + + + + + + + +
Mise à jour
Taille du fichier
Nombre de produits
-
- - - - - - Télécharger + +
+

Dataset GTIN

+ + + + + + + + + + + + +
+ + Voir les détails et télécharger + +
+ +
+

Dataset ISBN

+ + + + + + + + + + + + +
+ + Voir les détails et télécharger
-
- - - -

Format des données

- - - -
-
- Colonnes -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
barcodeLe code barre du produit, qui peut être un GTIN, EAN13 ou ISBN
brandMarque du produit
modelRéference constructeur du produit
last_updatedDernière mise à jour de ce produit, au format epoch en millisecondes
gs1_countryPays de l'organisme GS1 qui a délivré le code barre pour ce produit (hors ISBN)
gtinTypeType du code barre. Peut être ISBN_13, GTIN_13, GTIN_8, GTIN_12
offers_countNombre d'offres commerciales pour ce produit
min_pricePrix le plus faible pour ce produit
min_price_compensationCompensation carbone reversée, correspondant au prix le plus faible pour ce produit
currencyDevise du prix le plus faible et de la compensation carbone associée
categoriesCatégories dans lesquelles ce produit a été rencontré
urlURL de la fiche produit, uniquement si des offres commerciales sont disponibles
- - - - - -
-
-
-
-
- - - +
+
+
+
+
+ + From f918e19164b200b30e242413aa2e6b27f6c540ff Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 2 Aug 2024 12:05:18 +0200 Subject: [PATCH 25/52] Separate processing of ISBN and GTIN datasets in opendata export --- .../ui/services/OpenDataService.java | 56 +++++++++---------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index bf08fc5cd..a0621efc4 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -23,6 +23,7 @@ import org.open4goods.dao.ProductRepository; import org.open4goods.exceptions.TechnicalException; import org.open4goods.helper.ThrottlingInputStream; +import org.open4goods.model.BarcodeType; import org.open4goods.model.constants.CacheConstants; import org.open4goods.model.product.Product; import org.open4goods.ui.config.yml.UiConfig; @@ -117,57 +118,54 @@ public void generateOpendata() { if (exportRunning.get()) { LOGGER.error("Opendata export is already running"); - } else { - exportRunning.set(true); + return; } + exportRunning.set(true); + ZipOutputStream zos = null; - try { + FileOutputStream fos = null; + try { uiConfig.tmpOpenDataFile().getParentFile().mkdirs(); - FileOutputStream fos = new FileOutputStream(uiConfig.tmpOpenDataFile()); + fos = new FileOutputStream(uiConfig.tmpOpenDataFile()); zos = new ZipOutputStream(fos); - String filename = "open4goods-full-gtin-dataset.csv"; - ZipEntry entry = new ZipEntry(filename); // create a zip entry and add it pageSize ZipOutputStream - zos.putNextEntry(entry); - - CSVWriter writer = new CSVWriter(new OutputStreamWriter(zos)); - - // Writing the header - writer.writeNext(header); - - // Fixing the count - AtomicLong count = new AtomicLong(); + // Process ISBN_13 + LOGGER.info("Starting process for ISBN_13"); + processAndAddToZip(zos, "open4goods-isbn-dataset.csv", BarcodeType.ISBN_13); - // Iterating on datas - LOGGER.info("Starting opendata export"); - aggregatedDataRepository.exportAll().forEach(e -> { - count.incrementAndGet(); - writer.writeNext(toEntry(e)); // write the contents - }); + // Process GTIN/EAN (excluding ISBN_13) + LOGGER.info("Starting process for GTIN/EAN excluding ISBN_13"); + processAndAddToZip(zos, "open4goods-gtin-dataset.csv", BarcodeType.ISBN_13, true); - - - writer.flush(); - zos.closeEntry(); + zos.close(); + fos.close(); // Moving the tmp file - FileUtils.deleteQuietly(uiConfig.openDataFile()); + if (uiConfig.openDataFile().exists()) { + FileUtils.deleteQuietly(uiConfig.openDataFile()); + } FileUtils.moveFile(uiConfig.tmpOpenDataFile(), uiConfig.openDataFile()); - exportRunning.set(false); - LOGGER.info("{} rows exported in opendata CSV file located at {}", count.get(), - uiConfig.openDataFile().getAbsolutePath()); + LOGGER.info("Opendata CSV files generated and zipped successfully."); } catch (Exception e) { LOGGER.error("Error while generating opendata set", e); } finally { IOUtils.closeQuietly(zos); + IOUtils.closeQuietly(fos); + exportRunning.set(false); } } + private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType) throws IOException { + processAndAddToZip(zos, filename, barcodeType, false); + } + + private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType, boolean invertCondition) throws IOException {} + /** * Convert an aggregateddata pageSize a csv row * From 43261cd16cb1fd970a3036ef9dd54f5d5f6955af Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 2 Aug 2024 12:09:12 +0200 Subject: [PATCH 26/52] Implement processAndAddToZip method --- .../ui/services/OpenDataService.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index a0621efc4..a123f111c 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -164,7 +164,27 @@ private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeTyp processAndAddToZip(zos, filename, barcodeType, false); } - private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType, boolean invertCondition) throws IOException {} + private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType, boolean invertCondition) throws IOException { + ZipEntry entry = new ZipEntry(filename); + zos.putNextEntry(entry); + CSVWriter writer = new CSVWriter(new OutputStreamWriter(zos)); + writer.writeNext(header); + + AtomicLong count = new AtomicLong(); + try { + aggregatedDataRepository.exportAll().filter(e -> + invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) + ).forEach(e -> { + count.incrementAndGet(); + writer.writeNext(toEntry(e)); + }); + writer.flush(); + zos.closeEntry(); + LOGGER.info("{} rows exported in {}.", count.get(), filename); + } catch (Exception e) { + LOGGER.error("Error during processing of {}: {}", filename, e.getMessage()); + } + } /** * Convert an aggregateddata pageSize a csv row From 66e74b03b3959610a0d4571d12518f7cbbad8fa4 Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 2 Aug 2024 17:54:27 +0200 Subject: [PATCH 27/52] Typo --- ui/src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/main/resources/application.yml b/ui/src/main/resources/application.yml index 909548a3d..c9d5abe00 100644 --- a/ui/src/main/resources/application.yml +++ b/ui/src/main/resources/application.yml @@ -113,7 +113,7 @@ blogConfig: imageGenerationConfig: prompt: | Create a simple icon of a {VERTICAL} that strictly adheres to the following guidelines: - - The background color must be fully (#FFFFFF). + - The background color must be fully white (#FFFFFF). - The white background does not contains any gradient or texture. - The icon should be flat, in a modern, minimalist style. - The lines must be clean and simple. From ecbc396b00695493c43d6fdb790622bc72738c96 Mon Sep 17 00:00:00 2001 From: scezen Date: Sun, 4 Aug 2024 15:00:03 +0200 Subject: [PATCH 28/52] Small fixes, comments --- .../ui/services/OpenDataService.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index a123f111c..ec3eec847 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -63,13 +63,14 @@ public class OpenDataService { private AtomicInteger concurrentDownloads = new AtomicInteger(0); - - + private static final String ISBN_DATASET_FILENAME = "open4goods-isbn-dataset.csv"; + private static final String GTIN_DATASET_FILENAME = "open4goods-gtin-dataset.csv"; @Autowired - public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig) { + public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig){ this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; + generateOpendata(); } /** @@ -81,10 +82,10 @@ public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiCo */ public InputStream limitedRateStream() throws TechnicalException, FileNotFoundException { - // TODO : pageNumber conf + // TODO : in conf RateLimiter rateLimiter = RateLimiter.create(DOWNLOAD_SPEED_KB * FileUtils.ONE_KB); - // TODO : pageNumber conf + // TODO : in conf if (concurrentDownloads.get() >= CONCURRENT_DOWNLOADS) { throw new TechnicalException("Too many requests "); } else { @@ -109,11 +110,11 @@ public void close() throws IOException { } /** - * Iterates over all aggregatedData pageSize generate the zipped opendata CSV file + * Iterates over all aggregated data to generate the zipped opendata CSV file. * - * TODO : Schedule pageNumber conf + * TODO : Schedule in conf */ - @Scheduled(initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) + //@Scheduled(initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) public void generateOpendata() { if (exportRunning.get()) { @@ -134,11 +135,11 @@ public void generateOpendata() { // Process ISBN_13 LOGGER.info("Starting process for ISBN_13"); - processAndAddToZip(zos, "open4goods-isbn-dataset.csv", BarcodeType.ISBN_13); + processAndAddToZip(zos, ISBN_DATASET_FILENAME, BarcodeType.ISBN_13); // Process GTIN/EAN (excluding ISBN_13) - LOGGER.info("Starting process for GTIN/EAN excluding ISBN_13"); - processAndAddToZip(zos, "open4goods-gtin-dataset.csv", BarcodeType.ISBN_13, true); + LOGGER.info("Starting process for GTIN/EAN"); + processAndAddToZip(zos, GTIN_DATASET_FILENAME, BarcodeType.ISBN_13, true); zos.close(); fos.close(); @@ -164,6 +165,11 @@ private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeTyp processAndAddToZip(zos, filename, barcodeType, false); } + /** + * Processes the data and adds it to the zip output stream. + * + * @param invertCondition whether to invert the condition for filtering the data + */ private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType, boolean invertCondition) throws IOException { ZipEntry entry = new ZipEntry(filename); zos.putNextEntry(entry); @@ -208,7 +214,6 @@ private String[] toEntry(Product data) { line[4] = String.valueOf(data.getLastChange()); // "gs1_country" line[5] = data.getGtinInfos().getCountry(); - // "upcType" line[6] = data.getGtinInfos().getUpcType().toString(); From bb972630dddcd01738cf23430a6dc343a7b5f2ad Mon Sep 17 00:00:00 2001 From: scezen Date: Sun, 4 Aug 2024 15:03:56 +0200 Subject: [PATCH 29/52] Re-enable scheduling --- .../main/java/org/open4goods/ui/services/OpenDataService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index ec3eec847..2892236c8 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -70,7 +70,6 @@ public class OpenDataService { public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig){ this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; - generateOpendata(); } /** @@ -114,7 +113,7 @@ public void close() throws IOException { * * TODO : Schedule in conf */ - //@Scheduled(initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) + @Scheduled(initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) public void generateOpendata() { if (exportRunning.get()) { From a686f5aa6926c7f247f735ef966420ff3f5c9668 Mon Sep 17 00:00:00 2001 From: scezen Date: Sun, 4 Aug 2024 22:11:16 +0200 Subject: [PATCH 30/52] Separate directory preparation from generateOpendata --- .../ui/services/OpenDataService.java | 41 ++++--------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 2892236c8..bf79d5469 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -115,47 +115,19 @@ public void close() throws IOException { */ @Scheduled(initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) public void generateOpendata() { - - if (exportRunning.get()) { + if (exportRunning.getAndSet(true)) { LOGGER.error("Opendata export is already running"); return; } - exportRunning.set(true); - - ZipOutputStream zos = null; - FileOutputStream fos = null; - try { - uiConfig.tmpOpenDataFile().getParentFile().mkdirs(); - - fos = new FileOutputStream(uiConfig.tmpOpenDataFile()); - zos = new ZipOutputStream(fos); - - // Process ISBN_13 - LOGGER.info("Starting process for ISBN_13"); - processAndAddToZip(zos, ISBN_DATASET_FILENAME, BarcodeType.ISBN_13); - - // Process GTIN/EAN (excluding ISBN_13) - LOGGER.info("Starting process for GTIN/EAN"); - processAndAddToZip(zos, GTIN_DATASET_FILENAME, BarcodeType.ISBN_13, true); - - zos.close(); - fos.close(); - - // Moving the tmp file - if (uiConfig.openDataFile().exists()) { - FileUtils.deleteQuietly(uiConfig.openDataFile()); - } - FileUtils.moveFile(uiConfig.tmpOpenDataFile(), uiConfig.openDataFile()); - + prepareDirectories(); + processDataFiles(); + moveTmpFilesToFinalDestination(); LOGGER.info("Opendata CSV files generated and zipped successfully."); - } catch (Exception e) { LOGGER.error("Error while generating opendata set", e); } finally { - IOUtils.closeQuietly(zos); - IOUtils.closeQuietly(fos); exportRunning.set(false); } } @@ -164,6 +136,11 @@ private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeTyp processAndAddToZip(zos, filename, barcodeType, false); } + private void prepareDirectories() throws IOException { + uiConfig.tmpIsbnZipFile().getParentFile().mkdirs(); + uiConfig.tmpGtinZipFile().getParentFile().mkdirs(); + } + /** * Processes the data and adds it to the zip output stream. * From 62640ab45aa5036ea1eeb8aeb7774e35f3030060 Mon Sep 17 00:00:00 2001 From: scezen Date: Sun, 4 Aug 2024 22:12:33 +0200 Subject: [PATCH 31/52] Extract file moving logic to separate method --- .../ui/services/OpenDataService.java | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index bf79d5469..40caaefda 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -1,12 +1,6 @@ package org.open4goods.ui.services; -import java.io.BufferedInputStream; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStreamWriter; +import java.io.*; import java.text.CharacterIterator; import java.text.StringCharacterIterator; import java.time.Instant; @@ -141,6 +135,27 @@ private void prepareDirectories() throws IOException { uiConfig.tmpGtinZipFile().getParentFile().mkdirs(); } + private void processDataFiles() throws IOException { + LOGGER.info("Starting process for ISBN_13"); + processAndCreateZip(ISBN_DATASET_FILENAME, BarcodeType.ISBN_13, uiConfig.tmpIsbnZipFile()); + + LOGGER.info("Starting process for GTIN/EAN"); + processAndCreateZip(GTIN_DATASET_FILENAME, BarcodeType.ISBN_13, uiConfig.tmpGtinZipFile(), true); + } + + private void moveTmpFilesToFinalDestination() throws IOException { + moveFile(uiConfig.tmpIsbnZipFile(), uiConfig.isbnZipFile()); + moveFile(uiConfig.tmpGtinZipFile(), uiConfig.gtinZipFile()); + } + + private void moveFile(File src, File dest) throws IOException { + if (dest.exists()) { + FileUtils.deleteQuietly(dest); + } + FileUtils.moveFile(src, dest); + } + + /** * Processes the data and adds it to the zip output stream. * From f9804a0f4700fc8416b937ade336e6b7d7261eab Mon Sep 17 00:00:00 2001 From: scezen Date: Sun, 4 Aug 2024 22:13:53 +0200 Subject: [PATCH 32/52] Use try-with-resources for automatic resource management --- .../ui/services/OpenDataService.java | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 40caaefda..c98f59c8a 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -126,8 +126,8 @@ public void generateOpendata() { } } - private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType) throws IOException { - processAndAddToZip(zos, filename, barcodeType, false); + private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile) throws IOException { + processAndCreateZip(filename, barcodeType, zipFile, false); } private void prepareDirectories() throws IOException { @@ -156,28 +156,25 @@ private void moveFile(File src, File dest) throws IOException { } - /** - * Processes the data and adds it to the zip output stream. - * - * @param invertCondition whether to invert the condition for filtering the data - */ - private void processAndAddToZip(ZipOutputStream zos, String filename, BarcodeType barcodeType, boolean invertCondition) throws IOException { - ZipEntry entry = new ZipEntry(filename); - zos.putNextEntry(entry); - CSVWriter writer = new CSVWriter(new OutputStreamWriter(zos)); - writer.writeNext(header); + private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile, boolean invertCondition) throws IOException { + try (FileOutputStream fos = new FileOutputStream(zipFile); + ZipOutputStream zos = new ZipOutputStream(fos); + CSVWriter writer = new CSVWriter(new OutputStreamWriter(zos))) { - AtomicLong count = new AtomicLong(); - try { - aggregatedDataRepository.exportAll().filter(e -> + ZipEntry entry = new ZipEntry(filename); + zos.putNextEntry(entry); + writer.writeNext(header); + + AtomicLong count = new AtomicLong(); + aggregatedDataRepository.exportAll().limit(500).filter(e -> invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) ).forEach(e -> { count.incrementAndGet(); writer.writeNext(toEntry(e)); }); - writer.flush(); - zos.closeEntry(); LOGGER.info("{} rows exported in {}.", count.get(), filename); + + zos.closeEntry(); } catch (Exception e) { LOGGER.error("Error during processing of {}: {}", filename, e.getMessage()); } From 0114e59c4897f584d92c7bbbe49626cfad0f08b5 Mon Sep 17 00:00:00 2001 From: scezen Date: Sun, 4 Aug 2024 22:18:16 +0200 Subject: [PATCH 33/52] Update OpenDataController + UiConfig --- .../open4goods/ui/config/yml/UiConfig.java | 15 +++++++++++-- .../ui/pages/OpenDataController.java | 21 +++++++++++++++++++ .../ui/services/OpenDataService.java | 3 +-- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/config/yml/UiConfig.java b/ui/src/main/java/org/open4goods/ui/config/yml/UiConfig.java index 7b457952a..f06ded001 100644 --- a/ui/src/main/java/org/open4goods/ui/config/yml/UiConfig.java +++ b/ui/src/main/java/org/open4goods/ui/config/yml/UiConfig.java @@ -246,10 +246,21 @@ public File openDataFile() { return new File(rootFolder + File.separator+"opendata"+File.separator+"full.zip"); } - public File tmpOpenDataFile() { - return new File(rootFolder + File.separator+"opendata"+File.separator+"full-tmp.zip"); + public File isbnZipFile() { + return new File(rootFolder + File.separator + "opendata" + File.separator + "isbn.zip"); } + public File tmpIsbnZipFile() { + return new File(rootFolder + File.separator + "opendata" + File.separator + "isbn-tmp.zip"); + } + + public File gtinZipFile() { + return new File(rootFolder + File.separator + "opendata" + File.separator + "gtin.zip"); + } + + public File tmpGtinZipFile() { + return new File(rootFolder + File.separator + "opendata" + File.separator + "gtin-tmp.zip"); + } public String getRootFolder() { return rootFolder; diff --git a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java index 377f44589..179dceb5e 100644 --- a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java +++ b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java @@ -2,6 +2,8 @@ import java.io.IOException; import java.io.InputStream; +import java.util.Arrays; +import java.util.List; import org.apache.commons.io.IOUtils; import org.open4goods.exceptions.TechnicalException; @@ -53,6 +55,15 @@ public OpenDataController(OpenDataService openDataService) { public SitemapEntry getExposedUrls() { return SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, DEFAULT_PATH, 0.3, ChangeFreq.YEARLY); } + + @Override + public List getMultipleExposedUrls() { + return Arrays.asList( + SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, DEFAULT_PATH, 0.3, ChangeFreq.YEARLY), + SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/gtin-open-data.zip", 0.3, ChangeFreq.YEARLY), + SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/isbn-open-data.zip", 0.3, ChangeFreq.YEARLY) + ); + } @GetMapping(value = {DEFAULT_PATH}) public ModelAndView opendata(final HttpServletRequest request) { @@ -64,6 +75,16 @@ public ModelAndView opendata(final HttpServletRequest request) { return ret; } + @GetMapping(path = "/opendata/gtin-open-data.zip") + public void downloadGtinData(final HttpServletResponse response) throws IOException { + + } + + @GetMapping(path = "/opendata/isbn-open-data.zip") + public void downloadIsbnData(final HttpServletResponse response) throws IOException { + + } + @GetMapping(path = "/opendata/gtin-open-data.zip") public void opensearch(final HttpServletResponse response) throws IOException { try (InputStream str = openDataService.limitedRateStream()){ diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index c98f59c8a..e0707cc50 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -155,7 +155,6 @@ private void moveFile(File src, File dest) throws IOException { FileUtils.moveFile(src, dest); } - private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile, boolean invertCondition) throws IOException { try (FileOutputStream fos = new FileOutputStream(zipFile); ZipOutputStream zos = new ZipOutputStream(fos); @@ -166,7 +165,7 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File writer.writeNext(header); AtomicLong count = new AtomicLong(); - aggregatedDataRepository.exportAll().limit(500).filter(e -> + aggregatedDataRepository.exportAll().filter(e -> invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) ).forEach(e -> { count.incrementAndGet(); From 28fd4fd78141dc0045e78b3b2af99664513f037c Mon Sep 17 00:00:00 2001 From: scezen Date: Mon, 5 Aug 2024 09:59:16 +0200 Subject: [PATCH 34/52] Add missing cacheable methods for ISBN and GTIN file info --- .../ui/services/OpenDataService.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index e0707cc50..751fbb933 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -233,22 +233,24 @@ public void decrementDownloadCounter() { concurrentDownloads.decrementAndGet(); } - /** - * Return the file size in human readable way - * @return - */ - @Cacheable(key = "#root.method.name", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) - public String fileSize() { - return humanReadableByteCountBin(uiConfig.openDataFile().length()); + @Cacheable(key = "#root.method.name + 'Isbn'", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) + public String isbnFileSize() { + return humanReadableByteCountBin(uiConfig.isbnZipFile().length()); } - /** - * - * @return the last update date - */ - @Cacheable(key = "#root.method.name", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) - public Date lastUpdate() { - return Date.from(Instant.ofEpochMilli(uiConfig.openDataFile().lastModified())); + @Cacheable(key = "#root.method.name + 'Gtin'", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) + public String gtinFileSize() { + return humanReadableByteCountBin(uiConfig.gtinZipFile().length()); + } + + @Cacheable(key = "#root.method.name + 'Isbn'", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) + public Date isbnLastUpdate() { + return Date.from(Instant.ofEpochMilli(uiConfig.isbnZipFile().lastModified())); + } + + @Cacheable(key = "#root.method.name + 'Gtin'", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) + public Date gtinLastUpdate() { + return Date.from(Instant.ofEpochMilli(uiConfig.gtinZipFile().lastModified())); } /** From 08ef5691434b72aa11e9c001459aaefb09868242 Mon Sep 17 00:00:00 2001 From: scezen Date: Mon, 5 Aug 2024 09:59:42 +0200 Subject: [PATCH 35/52] Update OpenDataController to handle multiple file types and improve clarity --- .../ui/pages/OpenDataController.java | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java index 179dceb5e..5d610017a 100644 --- a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java +++ b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java @@ -1,5 +1,7 @@ package org.open4goods.ui.controllers.ui.pages; +import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.util.Arrays; @@ -7,6 +9,7 @@ import org.apache.commons.io.IOUtils; import org.open4goods.exceptions.TechnicalException; +import org.open4goods.ui.config.yml.UiConfig; import org.open4goods.ui.controllers.ui.UiService; import org.open4goods.ui.services.OpenDataService; import org.slf4j.Logger; @@ -38,8 +41,13 @@ public class OpenDataController implements SitemapExposedController{ // The siteConfig private final OpenDataService openDataService; private @Autowired UiService uiService; - public OpenDataController(OpenDataService openDataService) { + private final UiConfig uiConfig; + + + @Autowired + public OpenDataController(OpenDataService openDataService, UiConfig uiConfig) { this.openDataService = openDataService; + this.uiConfig = uiConfig; } /** @@ -64,39 +72,37 @@ public List getMultipleExposedUrls() { SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/isbn-open-data.zip", 0.3, ChangeFreq.YEARLY) ); } - - @GetMapping(value = {DEFAULT_PATH}) + + @GetMapping(value = {DEFAULT_PATH}) public ModelAndView opendata(final HttpServletRequest request) { final ModelAndView ret = uiService.defaultModelAndView("opendata", request); ret.addObject("count", openDataService.totalItems()); - ret.addObject("lastUpdated", openDataService.lastUpdate()); - ret.addObject("fileSize", openDataService.fileSize()); - ret.addObject("page","open data"); + ret.addObject("isbnLastUpdated", openDataService.isbnLastUpdate()); + ret.addObject("isbnFileSize", openDataService.isbnFileSize()); + ret.addObject("gtinLastUpdated", openDataService.gtinLastUpdate()); + ret.addObject("gtinFileSize", openDataService.gtinFileSize()); + ret.addObject("page", "open data"); return ret; } @GetMapping(path = "/opendata/gtin-open-data.zip") public void downloadGtinData(final HttpServletResponse response) throws IOException { - + downloadData(response, "gtin-open-data.zip", uiConfig.gtinZipFile()); } @GetMapping(path = "/opendata/isbn-open-data.zip") public void downloadIsbnData(final HttpServletResponse response) throws IOException { - + downloadData(response, "isbn-open-data.zip", uiConfig.isbnZipFile()); } - @GetMapping(path = "/opendata/gtin-open-data.zip") - public void opensearch(final HttpServletResponse response) throws IOException { - try (InputStream str = openDataService.limitedRateStream()){ + private void downloadData(final HttpServletResponse response, String fileName, File zipFile) throws IOException { + try (InputStream str = new FileInputStream(zipFile)) { response.setHeader("Content-type", "application/octet-stream"); - response.setHeader("Content-Disposition", "attachment; filename=\"gtin-open-data.zip\""); + response.setHeader("Content-Disposition", "attachment; filename=\"" + fileName + "\""); IOUtils.copy(str, response.getOutputStream()); } catch (IOException e) { - LOGGER.error("opendata file download error or interruption : {}",e.getMessage()); + LOGGER.error("opendata file download error or interruption : {}", e.getMessage()); openDataService.decrementDownloadCounter(); - } catch (TechnicalException e) { - response.sendError(429, "Exceding the " + OpenDataService.CONCURRENT_DOWNLOADS + " concurrent downloads availlable"); - LOGGER.error("opendata file download error : {}",e.getMessage()); } } From ad7b87203892bc54e2d73246fd35aa8c70540938 Mon Sep 17 00:00:00 2001 From: scezen Date: Mon, 5 Aug 2024 10:09:07 +0200 Subject: [PATCH 36/52] Fix ZIP entry handling in processAndCreateZip --- .../java/org/open4goods/ui/services/OpenDataService.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 751fbb933..90eb915a6 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -64,6 +64,7 @@ public class OpenDataService { public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig){ this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; + generateOpendata(); } /** @@ -165,15 +166,17 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File writer.writeNext(header); AtomicLong count = new AtomicLong(); - aggregatedDataRepository.exportAll().filter(e -> + aggregatedDataRepository.exportAll().limit(500).filter(e -> invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) ).forEach(e -> { count.incrementAndGet(); writer.writeNext(toEntry(e)); }); + writer.flush(); + zos.closeEntry(); // Ensure the entry is closed before ending the try block + LOGGER.info("{} rows exported in {}.", count.get(), filename); - zos.closeEntry(); } catch (Exception e) { LOGGER.error("Error during processing of {}: {}", filename, e.getMessage()); } From 95ddb1703263802b56a4af226ee3b8c2bab6da73 Mon Sep 17 00:00:00 2001 From: scezen Date: Mon, 5 Aug 2024 11:17:40 +0200 Subject: [PATCH 37/52] Add GTIN and ISBN specific pages + Controller Update --- .../ui/pages/OpenDataController.java | 22 +++++++++++++++++-- .../ui/services/OpenDataService.java | 1 - .../resources/templates/opendata-gtin.html | 12 ++++++++++ .../resources/templates/opendata-isbn.html | 12 ++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 ui/src/main/resources/templates/opendata-gtin.html create mode 100644 ui/src/main/resources/templates/opendata-isbn.html diff --git a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java index 5d610017a..5e71061b8 100644 --- a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java +++ b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java @@ -68,8 +68,8 @@ public SitemapEntry getExposedUrls() { public List getMultipleExposedUrls() { return Arrays.asList( SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, DEFAULT_PATH, 0.3, ChangeFreq.YEARLY), - SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/gtin-open-data.zip", 0.3, ChangeFreq.YEARLY), - SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/isbn-open-data.zip", 0.3, ChangeFreq.YEARLY) + SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/gtin", 0.3, ChangeFreq.YEARLY), + SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, "/opendata/isbn", 0.3, ChangeFreq.YEARLY) ); } @@ -85,6 +85,24 @@ public ModelAndView opendata(final HttpServletRequest request) { return ret; } + @GetMapping(value = {DEFAULT_PATH + "/gtin"}) + public ModelAndView opendataGtin(final HttpServletRequest request) { + final ModelAndView ret = uiService.defaultModelAndView("opendata-gtin", request); + ret.addObject("lastUpdated", openDataService.gtinLastUpdate()); + ret.addObject("fileSize", openDataService.gtinFileSize()); + ret.addObject("page", "gtin data"); + return ret; + } + + @GetMapping(value = {DEFAULT_PATH + "/isbn"}) + public ModelAndView opendataIsbn(final HttpServletRequest request) { + final ModelAndView ret = uiService.defaultModelAndView("opendata-isbn", request); + ret.addObject("lastUpdated", openDataService.isbnLastUpdate()); + ret.addObject("fileSize", openDataService.isbnFileSize()); + ret.addObject("page", "isbn data"); + return ret; + } + @GetMapping(path = "/opendata/gtin-open-data.zip") public void downloadGtinData(final HttpServletResponse response) throws IOException { downloadData(response, "gtin-open-data.zip", uiConfig.gtinZipFile()); diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 90eb915a6..6d6b47b25 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -64,7 +64,6 @@ public class OpenDataService { public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig){ this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; - generateOpendata(); } /** diff --git a/ui/src/main/resources/templates/opendata-gtin.html b/ui/src/main/resources/templates/opendata-gtin.html new file mode 100644 index 000000000..ff9df8bfa --- /dev/null +++ b/ui/src/main/resources/templates/opendata-gtin.html @@ -0,0 +1,12 @@ + + + + GTIN Open Data + + +

GTIN Open Data

+

Last Updated:

+

File Size:

+Download GTIN Data + + diff --git a/ui/src/main/resources/templates/opendata-isbn.html b/ui/src/main/resources/templates/opendata-isbn.html new file mode 100644 index 000000000..76156d78b --- /dev/null +++ b/ui/src/main/resources/templates/opendata-isbn.html @@ -0,0 +1,12 @@ + + + + ISBN Open Data + + +

ISBN Open Data

+

Last Updated:

+

File Size:

+Download ISBN Data + + From 8a1313de3b4b1e9290a5f78e0f56c66eae9b1194 Mon Sep 17 00:00:00 2001 From: goulven Date: Tue, 6 Aug 2024 15:01:42 +0200 Subject: [PATCH 38/52] Notes @scezen --- .../ui/services/OpenDataService.java | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 6d6b47b25..36dfa9be3 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -35,15 +35,35 @@ * * @author Goulven.Furet * + * TODO : Methods comments + * + * + * TODO : Add the below attributes only for ISBN dataset + * + * CLASSIFICATION DECITRE 1 Loisirs et Jeux 1 +CLASSIFICATION DECITRE 2 Jeux et activités 1 +CLASSIFICATION DECITRE 3 Coloriage Gommettes Autocollants +EDITEUR +FORMAT + +NB DE PAGES +SOUSCATEGORIE Enfant-jeunesse 1 +SOUSCATEGORIE2 Sport-et-loisirs + * + * */ public class OpenDataService { + private static final Logger LOGGER = LoggerFactory.getLogger(OpenDataService.class); + // Allowed download speed in kb + // TODO : In config (OpenDataConfig) private static final int DOWNLOAD_SPEED_KB = 256; - public static final int CONCURRENT_DOWNLOADS = 4; - private static final Logger LOGGER = LoggerFactory.getLogger(OpenDataService.class); + + private static final String ISBN_DATASET_FILENAME = "open4goods-isbn-dataset.csv"; + private static final String GTIN_DATASET_FILENAME = "open4goods-gtin-dataset.csv"; // The headers private static final String[] header = { "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", @@ -55,12 +75,9 @@ public class OpenDataService { // The flag that indicates wether opendata export is running or not private AtomicBoolean exportRunning = new AtomicBoolean(false); - private AtomicInteger concurrentDownloads = new AtomicInteger(0); + private AtomicInteger concurrentDownloadsCounter = new AtomicInteger(0); - private static final String ISBN_DATASET_FILENAME = "open4goods-isbn-dataset.csv"; - private static final String GTIN_DATASET_FILENAME = "open4goods-gtin-dataset.csv"; - @Autowired public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig){ this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; @@ -79,10 +96,10 @@ public InputStream limitedRateStream() throws TechnicalException, FileNotFoundEx RateLimiter rateLimiter = RateLimiter.create(DOWNLOAD_SPEED_KB * FileUtils.ONE_KB); // TODO : in conf - if (concurrentDownloads.get() >= CONCURRENT_DOWNLOADS) { + if (concurrentDownloadsCounter.get() >= CONCURRENT_DOWNLOADS) { throw new TechnicalException("Too many requests "); } else { - concurrentDownloads.incrementAndGet(); + concurrentDownloadsCounter.incrementAndGet(); } try { @@ -92,12 +109,12 @@ public InputStream limitedRateStream() throws TechnicalException, FileNotFoundEx @Override public void close() throws IOException { super.close(); - concurrentDownloads.decrementAndGet(); + concurrentDownloadsCounter.decrementAndGet(); LOGGER.info("Ending opendata dataset download"); } }; } catch (IOException e) { - concurrentDownloads.decrementAndGet(); + concurrentDownloadsCounter.decrementAndGet(); throw e; } } @@ -107,7 +124,7 @@ public void close() throws IOException { * * TODO : Schedule in conf */ - @Scheduled(initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) + @Scheduled( initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) public void generateOpendata() { if (exportRunning.getAndSet(true)) { LOGGER.error("Opendata export is already running"); @@ -165,6 +182,7 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File writer.writeNext(header); AtomicLong count = new AtomicLong(); + // TODO : Remove before MEP aggregatedDataRepository.exportAll().limit(500).filter(e -> invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) ).forEach(e -> { @@ -172,6 +190,8 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File writer.writeNext(toEntry(e)); }); writer.flush(); + + zos.closeEntry(); // Ensure the entry is closed before ending the try block LOGGER.info("{} rows exported in {}.", count.get(), filename); @@ -232,7 +252,7 @@ private String[] toEntry(Product data) { * (user stop the download) */ public void decrementDownloadCounter() { - concurrentDownloads.decrementAndGet(); + concurrentDownloadsCounter.decrementAndGet(); } @Cacheable(key = "#root.method.name + 'Isbn'", cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) From dfa068c8e0a7cc13d56b3867717c0e22bee6595b Mon Sep 17 00:00:00 2001 From: scezen Date: Wed, 7 Aug 2024 22:56:04 +0200 Subject: [PATCH 39/52] Typo --- ui/src/main/resources/templates/opendata.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/main/resources/templates/opendata.html b/ui/src/main/resources/templates/opendata.html index 27ef4d80d..0fdbfdd13 100644 --- a/ui/src/main/resources/templates/opendata.html +++ b/ui/src/main/resources/templates/opendata.html @@ -7,7 +7,7 @@ - Base de données ISBN en libre accès | Nudger + Base de données GTIN, EAN et ISBN en libre accès | Nudger From 369210b9f4172e912eae9ddd190a081ae382cd40 Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 9 Aug 2024 13:53:34 +0200 Subject: [PATCH 40/52] Adding OpenDataConfig --- .../org/open4goods/ui/config/AppConfig.java | 10 +++-- .../open4goods/ui/config/OpenDataConfig.java | 44 +++++++++++++++++++ ui/src/main/resources/application.yml | 10 ++++- 3 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java diff --git a/ui/src/main/java/org/open4goods/ui/config/AppConfig.java b/ui/src/main/java/org/open4goods/ui/config/AppConfig.java index ec5a00493..8bb47d221 100644 --- a/ui/src/main/java/org/open4goods/ui/config/AppConfig.java +++ b/ui/src/main/java/org/open4goods/ui/config/AppConfig.java @@ -180,11 +180,13 @@ IcecatService icecatFeatureService(UiConfig properties, RemoteFileCachingService // TODO : xmlMapper not injected because corruct the springdoc used one. Should use a @Primary derivation return new IcecatService(new XmlMapper(), properties.getIcecatFeatureConfig(), fileCachingService, properties.getRemoteCachingFolder(), brandService, verticalConfigService); } - - + + @Bean - OpenDataService openDataService(@Autowired ProductRepository aggregatedDataRepository, @Autowired UiConfig props) { - return new OpenDataService(aggregatedDataRepository, props); + OpenDataService openDataService(@Autowired ProductRepository aggregatedDataRepository, + @Autowired UiConfig props, + @Autowired OpenDataConfig openDataConfig) { + return new OpenDataService(aggregatedDataRepository, props, openDataConfig); } diff --git a/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java b/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java new file mode 100644 index 000000000..9f9cd8304 --- /dev/null +++ b/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java @@ -0,0 +1,44 @@ +package org.open4goods.ui.config; + +import org.springframework.context.annotation.Configuration; + +@Configuration +public class OpenDataConfig { + + private int downloadSpeedKb; + private int concurrentDownloads; + private long initialDelay; + private long fixedDelay; + + public int getDownloadSpeedKb() { + return downloadSpeedKb; + } + + public void setDownloadSpeedKb(int downloadSpeedKb) { + this.downloadSpeedKb = downloadSpeedKb; + } + + public int getConcurrentDownloads() { + return concurrentDownloads; + } + + public void setConcurrentDownloads(int concurrentDownloads) { + this.concurrentDownloads = concurrentDownloads; + } + + public long getInitialDelay() { + return initialDelay; + } + + public void setInitialDelay(long initialDelay) { + this.initialDelay = initialDelay; + } + + public long getFixedDelay() { + return fixedDelay; + } + + public void setFixedDelay(long fixedDelay) { + this.fixedDelay = fixedDelay; + } +} diff --git a/ui/src/main/resources/application.yml b/ui/src/main/resources/application.yml index c9d5abe00..4e60b6cf9 100644 --- a/ui/src/main/resources/application.yml +++ b/ui/src/main/resources/application.yml @@ -79,9 +79,15 @@ apiConfig: # log-responses: false -tagListUrl: https://open4good.github.io/open4goods/maven/taglist/taglist.xml +tagListUrl: https://open4good.github.io/open4goods/maven/taglist/taglist.xml + +openDataConfig: + downloadSpeedKb: 256 + concurrentDownloads: 4 + initialDelay: 3600000 # 1 heure + fixedDelay: 604800000 # 1 semaine + - feedbackConfig: githubConfig: accessToken: GITHUB_ACCESS_TOKEN From 562a43e0379fd37dd74cd959ee96f4a7210bc247 Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 9 Aug 2024 13:55:17 +0200 Subject: [PATCH 41/52] Frontend Tweaks + ISBN Headers --- .../ui/pages/OpenDataController.java | 2 + .../ui/services/OpenDataService.java | 97 ++++++--- .../resources/templates/opendata-gtin.html | 201 +++++++++++++++++- ui/src/main/resources/templates/opendata.html | 2 +- 4 files changed, 263 insertions(+), 39 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java index 5e71061b8..beb840960 100644 --- a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java +++ b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java @@ -89,6 +89,7 @@ public ModelAndView opendata(final HttpServletRequest request) { public ModelAndView opendataGtin(final HttpServletRequest request) { final ModelAndView ret = uiService.defaultModelAndView("opendata-gtin", request); ret.addObject("lastUpdated", openDataService.gtinLastUpdate()); + ret.addObject("countGTIN", openDataService.totalItemsGTIN()); ret.addObject("fileSize", openDataService.gtinFileSize()); ret.addObject("page", "gtin data"); return ret; @@ -98,6 +99,7 @@ public ModelAndView opendataGtin(final HttpServletRequest request) { public ModelAndView opendataIsbn(final HttpServletRequest request) { final ModelAndView ret = uiService.defaultModelAndView("opendata-isbn", request); ret.addObject("lastUpdated", openDataService.isbnLastUpdate()); + ret.addObject("countISBN", openDataService.totalItemsISBN()); ret.addObject("fileSize", openDataService.isbnFileSize()); ret.addObject("page", "isbn data"); return ret; diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 36dfa9be3..4f9ea302d 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -20,6 +20,7 @@ import org.open4goods.model.BarcodeType; import org.open4goods.model.constants.CacheConstants; import org.open4goods.model.product.Product; +import org.open4goods.ui.config.OpenDataConfig; import org.open4goods.ui.config.yml.UiConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,21 +57,27 @@ public class OpenDataService { private static final Logger LOGGER = LoggerFactory.getLogger(OpenDataService.class); - // Allowed download speed in kb - // TODO : In config (OpenDataConfig) - private static final int DOWNLOAD_SPEED_KB = 256; - public static final int CONCURRENT_DOWNLOADS = 4; +// // Allowed download speed in kb +// // TODO : In config (OpenDataConfig) +// private static final int DOWNLOAD_SPEED_KB = 256; +// public static final int CONCURRENT_DOWNLOADS = 4; private static final String ISBN_DATASET_FILENAME = "open4goods-isbn-dataset.csv"; private static final String GTIN_DATASET_FILENAME = "open4goods-gtin-dataset.csv"; - // The headers - private static final String[] header = { "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", + // GTIN headers + private static final String[] gtinHeader = { "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", "offers_count", "min_price", "min_price_compensation", "currency", "categories", "url" }; + // ISBN headers + private static final String[] isbnHeader = { "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", + "offers_count", "min_price", "min_price_compensation", "currency", "categories", "url", "editeur", "format" }; + + private ProductRepository aggregatedDataRepository; private UiConfig uiConfig; + private final OpenDataConfig openDataConfig; // The flag that indicates wether opendata export is running or not private AtomicBoolean exportRunning = new AtomicBoolean(false); @@ -78,9 +85,11 @@ public class OpenDataService { private AtomicInteger concurrentDownloadsCounter = new AtomicInteger(0); - public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig){ + public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiConfig, OpenDataConfig openDataConfig){ this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; + this.openDataConfig = openDataConfig; + generateOpendata(); } /** @@ -93,10 +102,10 @@ public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiCo public InputStream limitedRateStream() throws TechnicalException, FileNotFoundException { // TODO : in conf - RateLimiter rateLimiter = RateLimiter.create(DOWNLOAD_SPEED_KB * FileUtils.ONE_KB); + RateLimiter rateLimiter = RateLimiter.create(openDataConfig.getDownloadSpeedKb() * FileUtils.ONE_KB); // TODO : in conf - if (concurrentDownloadsCounter.get() >= CONCURRENT_DOWNLOADS) { + if (concurrentDownloadsCounter.get() >= openDataConfig.getConcurrentDownloads()) { throw new TechnicalException("Too many requests "); } else { concurrentDownloadsCounter.incrementAndGet(); @@ -124,7 +133,7 @@ public void close() throws IOException { * * TODO : Schedule in conf */ - @Scheduled( initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) + //@Scheduled( initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) public void generateOpendata() { if (exportRunning.getAndSet(true)) { LOGGER.error("Opendata export is already running"); @@ -179,20 +188,23 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File ZipEntry entry = new ZipEntry(filename); zos.putNextEntry(entry); - writer.writeNext(header); + + // Correct header if GTIN or ISBN + if (barcodeType == BarcodeType.ISBN_13) { + writer.writeNext(isbnHeader); + } else { + writer.writeNext(gtinHeader); + } AtomicLong count = new AtomicLong(); - // TODO : Remove before MEP aggregatedDataRepository.exportAll().limit(500).filter(e -> invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) ).forEach(e -> { count.incrementAndGet(); - writer.writeNext(toEntry(e)); + writer.writeNext(toEntry(e, barcodeType == BarcodeType.ISBN_13)); }); writer.flush(); - - - zos.closeEntry(); // Ensure the entry is closed before ending the try block + zos.closeEntry(); // entry is closed before ending the try block LOGGER.info("{} rows exported in {}.", count.get(), filename); @@ -201,35 +213,40 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File } } + /** * Convert an aggregateddata pageSize a csv row * * @param data * @return */ - private String[] toEntry(Product data) { + private String[] toEntry(Product data, boolean isIsbn) { + String[] line; - String[] line = new String[header.length]; + // Adapter la taille du tableau au nombre de headers + if (isIsbn) { + line = new String[isbnHeader.length]; + } else { + line = new String[gtinHeader.length]; + } - // "gtin" + // "gtin" line[0] = data.gtin(); - // "brand" + // "brand" line[1] = data.brand(); - // "model" + // "model" line[2] = data.model(); - // "shortest_name" + // "shortest_name" line[3] = data.getNames().shortestOfferName(); - // "last_updated" + // "last_updated" line[4] = String.valueOf(data.getLastChange()); - // "gs1_country" + // "gs1_country" line[5] = data.getGtinInfos().getCountry(); - // "upcType" + // "upcType" line[6] = data.getGtinInfos().getUpcType().toString(); - - - // "offers_count" + // "offers_count" line[7] = String.valueOf(data.getOffersCount()); - // "min_price" + // "min_price" if (null != data.bestPrice()) { line[8] = String.valueOf(data.bestPrice().getPrice()); // "compensation" @@ -237,16 +254,21 @@ private String[] toEntry(Product data) { // "currency" line[10] = data.bestPrice().getCurrency().toString(); // "url" - // TODO(gof) : i18n the url line[12] = ""; //uiConfig.getBaseUrl(Locale.FRANCE) + data.getNames().getName(); } - - // Categories + // "Categories" line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); + // Modifier classe Product +// if (isIsbn) { +// line[13] = data.getEditeur(); +// line[14] = data.getFormat(); +// } + return line; } + /** * Used pageSize decrement the download counter, for instance when IOexception occurs * (user stop the download) @@ -275,6 +297,17 @@ public Date gtinLastUpdate() { return Date.from(Instant.ofEpochMilli(uiConfig.gtinZipFile().lastModified())); } + @Cacheable(key = "#root.method.name + 'Isbn'", cacheNames = CacheConstants.ONE_DAY_LOCAL_CACHE_NAME) + public long totalItemsISBN() { + return aggregatedDataRepository.countItemsByBarcodeType(BarcodeType.ISBN_13); + } + + @Cacheable(key = "#root.method.name + 'Gtin'", cacheNames = CacheConstants.ONE_DAY_LOCAL_CACHE_NAME) + public long totalItemsGTIN() { + return aggregatedDataRepository.countItemsByBarcodeType( + BarcodeType.GTIN_8, BarcodeType.GTIN_12, BarcodeType.GTIN_13, BarcodeType.GTIN_14); + } + /** * * @return number of items diff --git a/ui/src/main/resources/templates/opendata-gtin.html b/ui/src/main/resources/templates/opendata-gtin.html index ff9df8bfa..a1ee654a4 100644 --- a/ui/src/main/resources/templates/opendata-gtin.html +++ b/ui/src/main/resources/templates/opendata-gtin.html @@ -1,12 +1,201 @@ - + + - GTIN Open Data + + Base de données GTIN en libre accès | Nudger + + + + + + + + + + + + -

GTIN Open Data

-

Last Updated:

-

File Size:

-Download GTIN Data + + +
+
+
+
+
+

Open Data : Base de données GTIN en libre accès

+
+
+
+
+
+
+
+
+
+
+
+ code barre en opendata +
+
+

Accédez à notre base de données GTIN. Ces données sont mises à jour une fois par semaine et proviennent de notre agrégation de contenu autour des catalogues d'affiliation.

+
+
+

Qu'est-ce qu'un GTIN ?

+

+ Le GTIN (Global Trade Item Number) est un identifiant unique utilisé mondialement pour identifier les produits et services. Il est crucial pour le commerce international et facilite la gestion des stocks, la logistique, et les transactions commerciales. +

+
    +
  •    Utilisé internationalement
    • +
  •    Facilite la gestion des stocks
    • +
  •    Optimise la logistique
    • +
+

L'importance des GTIN pour l'écologie

+

+ Les GTIN jouent un rôle clé dans la traçabilité des produits, ce qui est essentiel pour une gestion écologique efficace. En permettant un suivi précis des produits tout au long de la chaîne d'approvisionnement, les GTIN aident à réduire le gaspillage et à optimiser l'utilisation des ressources. +

+
    +
  •    Réduction du gaspillage
    • +
  •    Optimisation des ressources
    • +
  •    Support à l'économie circulaire
    • +
+

Conditions d'utilisation

+
    +
  • Ces données sont délivrées sous licence ouverte, et sont donc librement utilisables, sans restrictions d'usages.
    • +
  • Vous devez indiquer la provenance des données, en mentionnant la page produit quand disponible, ou en ajoutant un lien vers la page d'accueil de nudger.fr.
    • +
+
+
+
+
+

Obtenir les données

+ + + + + + + + + + + + + + + +
Mise à jour
Taille du fichier
Nombre de produits
+
+
+ + + + + Télécharger + +
+
+
+
+ +

Format des données

+ +
+
+ Colonnes +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
barcodeLe code barre GTIN du produit
brandMarque du produit
modelRéférence constructeur du produit
last_updatedDernière mise à jour de ce produit, au format epoch en millisecondes
gs1_countryPays de l'organisme GS1 qui a délivré le code barre pour ce produit
gtinTypeType du code barre. Peut être GTIN_13, GTIN_8, GTIN_12, GTIN_14
offers_countNombre d'offres commerciales pour ce produit
min_pricePrix le plus faible pour ce produit
min_price_compensationCompensation carbone reversée, correspondant au prix le plus faible pour ce produit
currencyDevise du prix le plus faible et de la compensation carbone associée
categoriesCatégories dans lesquelles ce produit a été rencontré
urlURL de la fiche produit, uniquement si des offres commerciales sont disponibles
+
+
+
+
+
+
+
+ + + + + + + + + + + diff --git a/ui/src/main/resources/templates/opendata.html b/ui/src/main/resources/templates/opendata.html index 0fdbfdd13..41d980ca7 100644 --- a/ui/src/main/resources/templates/opendata.html +++ b/ui/src/main/resources/templates/opendata.html @@ -78,7 +78,7 @@

Obtenir les données

- + From dcdb26f6772b7f3ad2dcb54cf81618ee66c9ca51 Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 9 Aug 2024 14:35:43 +0200 Subject: [PATCH 42/52] Fix header logic and filtering for ISBN and GTIN datasets --- .../ui/services/OpenDataService.java | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 4f9ea302d..e012a8a0b 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -181,7 +181,7 @@ private void moveFile(File src, File dest) throws IOException { FileUtils.moveFile(src, dest); } - private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile, boolean invertCondition) throws IOException { + private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile, boolean isGtinFile) throws IOException { try (FileOutputStream fos = new FileOutputStream(zipFile); ZipOutputStream zos = new ZipOutputStream(fos); CSVWriter writer = new CSVWriter(new OutputStreamWriter(zos))) { @@ -189,22 +189,27 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File ZipEntry entry = new ZipEntry(filename); zos.putNextEntry(entry); - // Correct header if GTIN or ISBN - if (barcodeType == BarcodeType.ISBN_13) { + // Sélection des en-têtes basés sur le type de fichier + if (!isGtinFile) { // Fichier ISBN writer.writeNext(isbnHeader); - } else { + } else { // Fichier GTIN writer.writeNext(gtinHeader); } AtomicLong count = new AtomicLong(); - aggregatedDataRepository.exportAll().limit(500).filter(e -> - invertCondition ? !e.getGtinInfos().getUpcType().equals(barcodeType) : e.getGtinInfos().getUpcType().equals(barcodeType) - ).forEach(e -> { + aggregatedDataRepository.exportAll().limit(1000).filter(e -> { + if (isGtinFile) { + return e.getGtinInfos().getUpcType() != BarcodeType.ISBN_13; + } else { + return e.getGtinInfos().getUpcType() == BarcodeType.ISBN_13; + } + }).forEach(e -> { count.incrementAndGet(); - writer.writeNext(toEntry(e, barcodeType == BarcodeType.ISBN_13)); + writer.writeNext(toEntry(e, !isGtinFile)); }); + writer.flush(); - zos.closeEntry(); // entry is closed before ending the try block + zos.closeEntry(); LOGGER.info("{} rows exported in {}.", count.get(), filename); @@ -214,6 +219,7 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File } + /** * Convert an aggregateddata pageSize a csv row * @@ -260,10 +266,10 @@ private String[] toEntry(Product data, boolean isIsbn) { line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); // Modifier classe Product -// if (isIsbn) { -// line[13] = data.getEditeur(); -// line[14] = data.getFormat(); -// } + if (isIsbn) { + line[13] = data.getAttributes().getReferentielAttributes().get("editeur"); + line[14] = data.getAttributes().getReferentielAttributes().get("format"); + } return line; } From 9c27756f82e1650134dae667ba7cf1611c702a1a Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 9 Aug 2024 14:37:21 +0200 Subject: [PATCH 43/52] Typo --- .../open4goods/ui/services/OpenDataService.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index e012a8a0b..a082b981c 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -189,10 +189,10 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File ZipEntry entry = new ZipEntry(filename); zos.putNextEntry(entry); - // Sélection des en-têtes basés sur le type de fichier - if (!isGtinFile) { // Fichier ISBN + // Headers selon type de fichier + if (!isGtinFile) { writer.writeNext(isbnHeader); - } else { // Fichier GTIN + } else { writer.writeNext(gtinHeader); } @@ -265,11 +265,11 @@ private String[] toEntry(Product data, boolean isIsbn) { // "Categories" line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); - // Modifier classe Product - if (isIsbn) { - line[13] = data.getAttributes().getReferentielAttributes().get("editeur"); - line[14] = data.getAttributes().getReferentielAttributes().get("format"); - } +// // Modifier classe Product +// if (isIsbn) { +// line[13] = data.getAttributes().getReferentielAttributes().get("editeur"); +// line[14] = data.getAttributes().getReferentielAttributes().get("format"); +// } return line; } From 8adf65c3b4946f7ecfb7c7e12cb99656b1da7a2a Mon Sep 17 00:00:00 2001 From: goulven Date: Fri, 9 Aug 2024 15:17:52 +0200 Subject: [PATCH 44/52] Removing scheduled props --- .../open4goods/ui/config/OpenDataConfig.java | 18 +----------------- ui/src/main/resources/application.yml | 2 -- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java b/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java index 9f9cd8304..e9af258f6 100644 --- a/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java +++ b/ui/src/main/java/org/open4goods/ui/config/OpenDataConfig.java @@ -7,8 +7,6 @@ public class OpenDataConfig { private int downloadSpeedKb; private int concurrentDownloads; - private long initialDelay; - private long fixedDelay; public int getDownloadSpeedKb() { return downloadSpeedKb; @@ -26,19 +24,5 @@ public void setConcurrentDownloads(int concurrentDownloads) { this.concurrentDownloads = concurrentDownloads; } - public long getInitialDelay() { - return initialDelay; - } - - public void setInitialDelay(long initialDelay) { - this.initialDelay = initialDelay; - } - - public long getFixedDelay() { - return fixedDelay; - } - - public void setFixedDelay(long fixedDelay) { - this.fixedDelay = fixedDelay; - } + } diff --git a/ui/src/main/resources/application.yml b/ui/src/main/resources/application.yml index 4e60b6cf9..587c4747a 100644 --- a/ui/src/main/resources/application.yml +++ b/ui/src/main/resources/application.yml @@ -84,8 +84,6 @@ tagListUrl: https://open4good.github.io/open4goods/maven/taglist/taglist.xml openDataConfig: downloadSpeedKb: 256 concurrentDownloads: 4 - initialDelay: 3600000 # 1 heure - fixedDelay: 604800000 # 1 semaine feedbackConfig: From d81a04ed3f1119acdbc43b54f885e1d88d424e10 Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 9 Aug 2024 15:20:15 +0200 Subject: [PATCH 45/52] ProductRepository --- .../main/java/org/open4goods/dao/ProductRepository.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/commons/src/main/java/org/open4goods/dao/ProductRepository.java b/commons/src/main/java/org/open4goods/dao/ProductRepository.java index 02cab4fc0..2a955c261 100644 --- a/commons/src/main/java/org/open4goods/dao/ProductRepository.java +++ b/commons/src/main/java/org/open4goods/dao/ProductRepository.java @@ -14,6 +14,7 @@ import org.open4goods.config.yml.ui.VerticalConfig; import org.open4goods.exceptions.ResourceNotFoundException; +import org.open4goods.model.BarcodeType; import org.open4goods.model.constants.CacheConstants; import org.open4goods.model.product.Product; import org.open4goods.store.repository.ProductIndexationWorker; @@ -496,6 +497,12 @@ public Long countMainIndex() { return elasticsearchTemplate.count(Query.findAll(), current_index); } + @Cacheable(cacheNames = CacheConstants.ONE_DAY_LOCAL_CACHE_NAME) + public long countItemsByBarcodeType(BarcodeType... barcodeTypes) { + Criteria criteria = new Criteria("gtinInfos.upcType").in((Object[]) barcodeTypes); + CriteriaQuery query = new CriteriaQuery(criteria); + return elasticsearchTemplate.count(query, current_index); + } @Cacheable(cacheNames = CacheConstants.ONE_HOUR_LOCAL_CACHE_NAME) public Long countMainIndexHavingPrice() { CriteriaQuery query = new CriteriaQuery(getValidDateQuery()); From 0cec9769302a545b22089735c34931263da60ad5 Mon Sep 17 00:00:00 2001 From: goulven Date: Fri, 9 Aug 2024 15:54:11 +0200 Subject: [PATCH 46/52] Working session --- .../org/open4goods/dao/ProductRepository.java | 15 +++ .../org/open4goods/model/BarcodeType.java | 7 +- .../model/product/AggregatedAttributes.java | 1 + .../ui/services/OpenDataService.java | 113 +++++++++++++++--- 4 files changed, 116 insertions(+), 20 deletions(-) diff --git a/commons/src/main/java/org/open4goods/dao/ProductRepository.java b/commons/src/main/java/org/open4goods/dao/ProductRepository.java index 2a955c261..a95eff3ee 100644 --- a/commons/src/main/java/org/open4goods/dao/ProductRepository.java +++ b/commons/src/main/java/org/open4goods/dao/ProductRepository.java @@ -125,6 +125,21 @@ public Stream exportAll() { .map(SearchHit::getContent); } + /** + * Export all aggregated data, corresponding to the given Barcodes + * + * @return + */ + public Stream exportAll(BarcodeType... barcodeTypes) { + + Criteria criteria = new Criteria("gtinInfos.upcType").in((Object[]) barcodeTypes); + CriteriaQuery query = new CriteriaQuery(criteria); + + return elasticsearchTemplate.searchForStream(query, Product.class, current_index).stream() + .map(SearchHit::getContent); + } + + public Stream searchInValidPrices(String query, final String indexName, int from, int to) { Criteria c = new Criteria().expression(query).and(getValidDateQuery()); diff --git a/commons/src/main/java/org/open4goods/model/BarcodeType.java b/commons/src/main/java/org/open4goods/model/BarcodeType.java index 29fee7da0..ef562e862 100644 --- a/commons/src/main/java/org/open4goods/model/BarcodeType.java +++ b/commons/src/main/java/org/open4goods/model/BarcodeType.java @@ -1,7 +1,12 @@ package org.open4goods.model; public enum BarcodeType { - ISBN_13, GTIN_13, GTIN_8, GTIN_12, GTIN_14, UNKNOWN, + ISBN_13, + GTIN_13, + GTIN_8, + GTIN_12, + GTIN_14, + UNKNOWN, } diff --git a/commons/src/main/java/org/open4goods/model/product/AggregatedAttributes.java b/commons/src/main/java/org/open4goods/model/product/AggregatedAttributes.java index d292ed168..b1a443274 100644 --- a/commons/src/main/java/org/open4goods/model/product/AggregatedAttributes.java +++ b/commons/src/main/java/org/open4goods/model/product/AggregatedAttributes.java @@ -24,6 +24,7 @@ public class AggregatedAttributes { //TODO: rename + // TODO : specifiy indexation rules private Map aggregatedAttributes = new HashMap<>(); diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index a082b981c..16a9188b8 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -189,23 +189,25 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File ZipEntry entry = new ZipEntry(filename); zos.putNextEntry(entry); + + BarcodeType[] types = null; + // Headers selon type de fichier - if (!isGtinFile) { - writer.writeNext(isbnHeader); - } else { + if (isGtinFile) { writer.writeNext(gtinHeader); + types = new BarcodeType[] { BarcodeType.GTIN_8, BarcodeType.GTIN_12, BarcodeType.GTIN_13, BarcodeType.GTIN_14 }; + } else { + writer.writeNext(isbnHeader); + types = new BarcodeType[] { BarcodeType.ISBN_13 }; } AtomicLong count = new AtomicLong(); - aggregatedDataRepository.exportAll().limit(1000).filter(e -> { - if (isGtinFile) { - return e.getGtinInfos().getUpcType() != BarcodeType.ISBN_13; - } else { - return e.getGtinInfos().getUpcType() == BarcodeType.ISBN_13; - } - }).forEach(e -> { - count.incrementAndGet(); - writer.writeNext(toEntry(e, !isGtinFile)); + //TODO : Remind to remove limit + aggregatedDataRepository.exportAll(types) + .limit(1000) + .forEach(e -> { + count.incrementAndGet(); + writer.writeNext(isGtinFile ? toGtinEntry(e) : toIsbnEntry(e)); }); writer.flush(); @@ -226,15 +228,57 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File * @param data * @return */ - private String[] toEntry(Product data, boolean isIsbn) { - String[] line; + private String[] toGtinEntry(Product data) { + String[] line = new String[gtinHeader.length]; - // Adapter la taille du tableau au nombre de headers - if (isIsbn) { - line = new String[isbnHeader.length]; - } else { - line = new String[gtinHeader.length]; + // "gtin" + line[0] = data.gtin(); + // "brand" + line[1] = data.brand(); + // "model" + line[2] = data.model(); + // "shortest_name" + line[3] = data.getNames().shortestOfferName(); + // "last_updated" + line[4] = String.valueOf(data.getLastChange()); + // "gs1_country" + line[5] = data.getGtinInfos().getCountry(); + // "upcType" + line[6] = data.getGtinInfos().getUpcType().toString(); + // "offers_count" + line[7] = String.valueOf(data.getOffersCount()); + // "min_price" + if (null != data.bestPrice()) { + line[8] = String.valueOf(data.bestPrice().getPrice()); + // "compensation" + line[9] = String.valueOf(data.bestPrice().getCompensation()); + // "currency" + line[10] = data.bestPrice().getCurrency().toString(); + // "url" + line[12] = ""; //uiConfig.getBaseUrl(Locale.FRANCE) + data.getNames().getName(); } + // "Categories" + line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); + +// // Modifier classe Product +// if (isIsbn) { +// line[13] = data.getAttributes().getReferentielAttributes().get("editeur"); +// line[14] = data.getAttributes().getReferentielAttributes().get("format"); +// } + + return line; + } + + + + /** + * Convert an aggregateddata pageSize a csv row + * + * @param data + * @return + */ + private String[] toIsbnEntry(Product data) { + String[] line = new String[isbnHeader.length]; // "gtin" line[0] = data.gtin(); @@ -264,6 +308,10 @@ private String[] toEntry(Product data, boolean isIsbn) { } // "Categories" line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); + + + + String attrVal = getAttribute(data, "editeur"); // // Modifier classe Product // if (isIsbn) { @@ -274,7 +322,34 @@ private String[] toEntry(Product data, boolean isIsbn) { return line; } + /** + * Try a direct access to aggregatedattributes, if fail iterate over unmapped ones + * TODO : Review when all attributes acessible by map + * @param data + * @param key + * @return + */ + private String getAttribute(Product data, String key) { + + // Direct check against aggregatedAttributes + String value = data.getAttributes().getAggregatedAttributes().get(key).getValue(); + + if (StringUtils.isEmpty(value)) { + // Checking in unmapped attributes + + value = data.getAttributes().getUnmapedAttributes().stream() + .filter(a -> a.getName().equalsIgnoreCase(key)) + .findFirst() + .map(a -> a.getValue()) + .orElse(null); + + } + return value; + } + + + /** * Used pageSize decrement the download counter, for instance when IOexception occurs * (user stop the download) From 3e3d7e548e78909ae6758a1dc04d4cf8341d4a87 Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 9 Aug 2024 21:06:50 +0200 Subject: [PATCH 47/52] Specific ISBN attributes and null values handling + Refactor and cleanup --- .../ui/pages/OpenDataController.java | 12 - .../ui/services/OpenDataService.java | 299 ++++++++---------- 2 files changed, 129 insertions(+), 182 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java index beb840960..ffa4ed8e1 100644 --- a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java +++ b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java @@ -35,30 +35,18 @@ public class OpenDataController implements SitemapExposedController{ public static final String DEFAULT_PATH="/opendata"; - private static final Logger LOGGER = LoggerFactory.getLogger(OpenDataController.class); - // The siteConfig private final OpenDataService openDataService; private @Autowired UiService uiService; private final UiConfig uiConfig; - @Autowired public OpenDataController(OpenDataService openDataService, UiConfig uiConfig) { this.openDataService = openDataService; this.uiConfig = uiConfig; } - /** - * The Home page. - * - * @param request - * @param response - * @return - * @throws UnirestException - */ - @Override public SitemapEntry getExposedUrls() { return SitemapEntry.of(SitemapEntry.LANGUAGE_DEFAULT, DEFAULT_PATH, 0.3, ChangeFreq.YEARLY); diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 16a9188b8..79e500aa4 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -5,6 +5,7 @@ import java.text.StringCharacterIterator; import java.time.Instant; import java.util.Date; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -19,6 +20,7 @@ import org.open4goods.helper.ThrottlingInputStream; import org.open4goods.model.BarcodeType; import org.open4goods.model.constants.CacheConstants; +import org.open4goods.model.product.AggregatedAttribute; import org.open4goods.model.product.Product; import org.open4goods.ui.config.OpenDataConfig; import org.open4goods.ui.config.yml.UiConfig; @@ -32,48 +34,29 @@ import com.opencsv.CSVWriter; /** - * Service in charge of generating the CSV opendata file - * - * @author Goulven.Furet - * - * TODO : Methods comments - * - * - * TODO : Add the below attributes only for ISBN dataset - * - * CLASSIFICATION DECITRE 1 Loisirs et Jeux 1 -CLASSIFICATION DECITRE 2 Jeux et activités 1 -CLASSIFICATION DECITRE 3 Coloriage Gommettes Autocollants -EDITEUR -FORMAT - -NB DE PAGES -SOUSCATEGORIE Enfant-jeunesse 1 -SOUSCATEGORIE2 Sport-et-loisirs - * - * + * Service responsible for generating and managing the CSV opendata files. + * Handles the creation of GTIN and ISBN datasets, their compression into ZIP files, + * and the management of download limits and rates. */ public class OpenDataService { private static final Logger LOGGER = LoggerFactory.getLogger(OpenDataService.class); -// // Allowed download speed in kb -// // TODO : In config (OpenDataConfig) -// private static final int DOWNLOAD_SPEED_KB = 256; -// public static final int CONCURRENT_DOWNLOADS = 4; - - private static final String ISBN_DATASET_FILENAME = "open4goods-isbn-dataset.csv"; private static final String GTIN_DATASET_FILENAME = "open4goods-gtin-dataset.csv"; - // GTIN headers - private static final String[] gtinHeader = { "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", - "offers_count", "min_price", "min_price_compensation", "currency", "categories", "url" }; - - // ISBN headers - private static final String[] isbnHeader = { "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", - "offers_count", "min_price", "min_price_compensation", "currency", "categories", "url", "editeur", "format" }; + private static final String[] GTIN_HEADER = { + "code", "brand", "model", "name", "last_updated", "gs1_country", "gtinType", + "offers_count", "min_price", "min_price_compensation", "currency", "categories", "url" + }; + private static final String[] ISBN_HEADER = { + "code", "brand", "model", "name", "last_updated", "gs1_country", "gtintype", + "offers_count", "min_price", "min_price_compensation", "currency", "categories", "url", + "editeur", "format", "nb de pages", + "classification decitre 1", "classification decitre 2", "classification decitre 3", + "souscategorie", "souscategorie2" + }; private ProductRepository aggregatedDataRepository; private UiConfig uiConfig; @@ -81,7 +64,6 @@ public class OpenDataService { // The flag that indicates wether opendata export is running or not private AtomicBoolean exportRunning = new AtomicBoolean(false); - private AtomicInteger concurrentDownloadsCounter = new AtomicInteger(0); @@ -93,18 +75,13 @@ public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiCo } /** - * - * @return a limited stream pageSize the opendata set. Limited in bandwith, and - * limited in number of conccurent downloads Limited - * @throws FileNotFoundException - * @throws TechnicalException + * Provides a limited bandwidth stream for downloading the opendata file. + * Enforces a limit on the number of concurrent downloads. */ public InputStream limitedRateStream() throws TechnicalException, FileNotFoundException { - // TODO : in conf RateLimiter rateLimiter = RateLimiter.create(openDataConfig.getDownloadSpeedKb() * FileUtils.ONE_KB); - // TODO : in conf if (concurrentDownloadsCounter.get() >= openDataConfig.getConcurrentDownloads()) { throw new TechnicalException("Too many requests "); } else { @@ -113,8 +90,7 @@ public InputStream limitedRateStream() throws TechnicalException, FileNotFoundEx try { LOGGER.info("Starting opendata dataset download"); - return new ThrottlingInputStream(new BufferedInputStream(new FileInputStream(uiConfig.openDataFile())), - rateLimiter) { + return new ThrottlingInputStream(new BufferedInputStream(new FileInputStream(uiConfig.openDataFile())), rateLimiter) { @Override public void close() throws IOException { super.close(); @@ -129,12 +105,13 @@ public void close() throws IOException { } /** - * Iterates over all aggregated data to generate the zipped opendata CSV file. - * + * Generates the opendata CSV files and compresses them into ZIP files. + * This method is scheduled to run periodically. * TODO : Schedule in conf */ - //@Scheduled( initialDelay = 1000L *3600, fixedDelay = 1000L * 3600 * 24 * 7) + @Scheduled(initialDelay = 1000L * 3600, fixedDelay = 1000L * 3600 * 24 * 7) public void generateOpendata() { + if (exportRunning.getAndSet(true)) { LOGGER.error("Opendata export is already running"); return; @@ -152,15 +129,17 @@ public void generateOpendata() { } } - private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile) throws IOException { - processAndCreateZip(filename, barcodeType, zipFile, false); - } - + /** + * Prepares the necessary directories for storing temporary files. + */ private void prepareDirectories() throws IOException { uiConfig.tmpIsbnZipFile().getParentFile().mkdirs(); uiConfig.tmpGtinZipFile().getParentFile().mkdirs(); } + /** + * Processes and creates the ZIP files for the opendata. + */ private void processDataFiles() throws IOException { LOGGER.info("Starting process for ISBN_13"); processAndCreateZip(ISBN_DATASET_FILENAME, BarcodeType.ISBN_13, uiConfig.tmpIsbnZipFile()); @@ -169,11 +148,17 @@ private void processDataFiles() throws IOException { processAndCreateZip(GTIN_DATASET_FILENAME, BarcodeType.ISBN_13, uiConfig.tmpGtinZipFile(), true); } + /** + * Moves the temporary files to their final destination. + */ private void moveTmpFilesToFinalDestination() throws IOException { moveFile(uiConfig.tmpIsbnZipFile(), uiConfig.isbnZipFile()); moveFile(uiConfig.tmpGtinZipFile(), uiConfig.gtinZipFile()); } + /** + * Moves a file from the source to the destination. + */ private void moveFile(File src, File dest) throws IOException { if (dest.exists()) { FileUtils.deleteQuietly(dest); @@ -181,6 +166,19 @@ private void moveFile(File src, File dest) throws IOException { FileUtils.moveFile(src, dest); } + private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile) throws IOException { + processAndCreateZip(filename, barcodeType, zipFile, false); + } + + /** + * Processes the data and creates a ZIP file for the specified barcode type. + * + * @param filename The name of the file to be created. + * @param barcodeType The type of barcode being processed. + * @param zipFile The file object representing the ZIP file to be created. + * @param isGtinFile Indicates if the file is a GTIN file. + * @throws IOException If there is an error during file creation or writing. + */ private void processAndCreateZip(String filename, BarcodeType barcodeType, File zipFile, boolean isGtinFile) throws IOException { try (FileOutputStream fos = new FileOutputStream(zipFile); ZipOutputStream zos = new ZipOutputStream(fos); @@ -189,26 +187,23 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File ZipEntry entry = new ZipEntry(filename); zos.putNextEntry(entry); - - BarcodeType[] types = null; - - // Headers selon type de fichier + // Set the appropriate header and barcode types + BarcodeType[] types; if (isGtinFile) { - writer.writeNext(gtinHeader); - types = new BarcodeType[] { BarcodeType.GTIN_8, BarcodeType.GTIN_12, BarcodeType.GTIN_13, BarcodeType.GTIN_14 }; + writer.writeNext(GTIN_HEADER); + types = new BarcodeType[]{BarcodeType.GTIN_8, BarcodeType.GTIN_12, BarcodeType.GTIN_13, BarcodeType.GTIN_14}; } else { - writer.writeNext(isbnHeader); - types = new BarcodeType[] { BarcodeType.ISBN_13 }; + writer.writeNext(ISBN_HEADER); + types = new BarcodeType[]{BarcodeType.ISBN_13}; } AtomicLong count = new AtomicLong(); - //TODO : Remind to remove limit + aggregatedDataRepository.exportAll(types) - .limit(1000) - .forEach(e -> { - count.incrementAndGet(); - writer.writeNext(isGtinFile ? toGtinEntry(e) : toIsbnEntry(e)); - }); + .forEach(e -> { + count.incrementAndGet(); + writer.writeNext(isGtinFile ? toGtinEntry(e) : toIsbnEntry(e)); + }); writer.flush(); zos.closeEntry(); @@ -223,136 +218,100 @@ private void processAndCreateZip(String filename, BarcodeType barcodeType, File /** - * Convert an aggregateddata pageSize a csv row - * - * @param data - * @return + * Converts a Product object into a CSV row for GTIN. */ private String[] toGtinEntry(Product data) { - String[] line = new String[gtinHeader.length]; - - // "gtin" - line[0] = data.gtin(); - // "brand" - line[1] = data.brand(); - // "model" - line[2] = data.model(); - // "shortest_name" - line[3] = data.getNames().shortestOfferName(); - // "last_updated" - line[4] = String.valueOf(data.getLastChange()); - // "gs1_country" - line[5] = data.getGtinInfos().getCountry(); - // "upcType" - line[6] = data.getGtinInfos().getUpcType().toString(); - // "offers_count" - line[7] = String.valueOf(data.getOffersCount()); - // "min_price" + String[] line = new String[GTIN_HEADER.length]; + + line[0] = data.gtin(); // "code" + line[1] = data.brand(); // "brand" + line[2] = data.model(); // "model" + line[3] = data.getNames().shortestOfferName(); // "name" + line[4] = String.valueOf(data.getLastChange()); // "last_updated" + line[5] = data.getGtinInfos().getCountry(); // "gs1_country" + line[6] = data.getGtinInfos().getUpcType().toString(); // "gtinType" + line[7] = String.valueOf(data.getOffersCount()); // "offers_count" + if (null != data.bestPrice()) { - line[8] = String.valueOf(data.bestPrice().getPrice()); - // "compensation" - line[9] = String.valueOf(data.bestPrice().getCompensation()); - // "currency" - line[10] = data.bestPrice().getCurrency().toString(); - // "url" - line[12] = ""; //uiConfig.getBaseUrl(Locale.FRANCE) + data.getNames().getName(); + line[8] = String.valueOf(data.bestPrice().getPrice()); // "min_price" + line[9] = String.valueOf(data.bestPrice().getCompensation()); // "min_price_compensation" + line[10] = data.bestPrice().getCurrency().toString(); // "currency" + line[12] = ""; // TODO: Construct the URL } - // "Categories" - line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); -// // Modifier classe Product -// if (isIsbn) { -// line[13] = data.getAttributes().getReferentielAttributes().get("editeur"); -// line[14] = data.getAttributes().getReferentielAttributes().get("format"); -// } + line[11] = StringUtils.join(data.getDatasourceCategories(), " ; "); // "categories" return line; } - - /** - * Convert an aggregateddata pageSize a csv row - * - * @param data - * @return + * Converts a Product object into a CSV row for ISBN. */ private String[] toIsbnEntry(Product data) { - String[] line = new String[isbnHeader.length]; - - // "gtin" - line[0] = data.gtin(); - // "brand" - line[1] = data.brand(); - // "model" - line[2] = data.model(); - // "shortest_name" - line[3] = data.getNames().shortestOfferName(); - // "last_updated" - line[4] = String.valueOf(data.getLastChange()); - // "gs1_country" - line[5] = data.getGtinInfos().getCountry(); - // "upcType" - line[6] = data.getGtinInfos().getUpcType().toString(); - // "offers_count" - line[7] = String.valueOf(data.getOffersCount()); - // "min_price" + String[] line = new String[ISBN_HEADER.length]; + + line[0] = data.gtin(); // "code" + line[1] = data.brand(); // "brand" + line[2] = data.model(); // "model" + line[3] = data.getNames().shortestOfferName(); // "name" + line[4] = String.valueOf(data.getLastChange()); // "last_updated" + line[5] = data.getGtinInfos().getCountry(); // "gs1_country" + line[6] = data.getGtinInfos().getUpcType().toString(); // "gtintype" + line[7] = String.valueOf(data.getOffersCount()); // "offers_count" + if (null != data.bestPrice()) { - line[8] = String.valueOf(data.bestPrice().getPrice()); - // "compensation" - line[9] = String.valueOf(data.bestPrice().getCompensation()); - // "currency" - line[10] = data.bestPrice().getCurrency().toString(); - // "url" - line[12] = ""; //uiConfig.getBaseUrl(Locale.FRANCE) + data.getNames().getName(); + line[8] = String.valueOf(data.bestPrice().getPrice()); // "min_price" + line[9] = String.valueOf(data.bestPrice().getCompensation()); // "min_price_compensation" + line[10] = data.bestPrice().getCurrency().toString(); // "currency" + line[12] = ""; // TODO: Construct the URL } - // "Categories" - line[11] = StringUtils.join(data.getDatasourceCategories()," ; "); - - - - String attrVal = getAttribute(data, "editeur"); - -// // Modifier classe Product -// if (isIsbn) { -// line[13] = data.getAttributes().getReferentielAttributes().get("editeur"); -// line[14] = data.getAttributes().getReferentielAttributes().get("format"); -// } + + line[11] = StringUtils.join(data.getDatasourceCategories(), " ; "); // "categories" + + line[13] = getAttribute(data, "EDITEUR"); + line[14] = getAttribute(data, "FORMAT"); + line[15] = getAttribute(data, "NB DE PAGES"); + line[16] = getAttribute(data, "CLASSIFICATION DECITRE 1"); + line[17] = getAttribute(data, "CLASSIFICATION DECITRE 2"); + line[18] = getAttribute(data, "CLASSIFICATION DECITRE 3"); + line[19] = getAttribute(data, "SOUSCATEGORIE"); + line[20] = getAttribute(data, "SOUSCATEGORIE2"); return line; } /** - * Try a direct access to aggregatedattributes, if fail iterate over unmapped ones + * Retrieves a specific attribute from the product data. * TODO : Review when all attributes acessible by map - * @param data - * @param key - * @return */ private String getAttribute(Product data, String key) { - - // Direct check against aggregatedAttributes - String value = data.getAttributes().getAggregatedAttributes().get(key).getValue(); - + + String value = null; + + Map aggregatedAttributes = data.getAttributes().getAggregatedAttributes(); + + if (aggregatedAttributes.containsKey(key)) { + AggregatedAttribute attribute = aggregatedAttributes.get(key); + + if (attribute != null) { + value = attribute.getValue(); + } + } + if (StringUtils.isEmpty(value)) { // Checking in unmapped attributes - value = data.getAttributes().getUnmapedAttributes().stream() - .filter(a -> a.getName().equalsIgnoreCase(key)) - .findFirst() - .map(a -> a.getValue()) - .orElse(null); - + .filter(a -> a.getName().equalsIgnoreCase(key)) + .findFirst() + .map(AggregatedAttribute::getValue) + .orElse(null); } return value; } - - - + /** - * Used pageSize decrement the download counter, for instance when IOexception occurs - * (user stop the download) + * Decrements the download counter, for instance when an IOException occurs (e.g., user stops the download). */ public void decrementDownloadCounter() { concurrentDownloadsCounter.decrementAndGet(); @@ -390,8 +349,9 @@ public long totalItemsGTIN() { } /** + * Retrieves the total number of items in the main index. * - * @return number of items + * @return The total number of items. */ @Cacheable(key = "#root.method.name", cacheNames = CacheConstants.ONE_DAY_LOCAL_CACHE_NAME) public long totalItems() { @@ -399,9 +359,10 @@ public long totalItems() { } /** - * Convert a size in human readable form - * @param bytes - * @return + * Converts a size in bytes to a human-readable format. + * + * @param bytes The size in bytes. + * @return The size in a human-readable format (e.g., "1.2 MB"). */ public static String humanReadableByteCountBin(long bytes) { if (-1000 < bytes && bytes < 1000) { @@ -414,6 +375,4 @@ public static String humanReadableByteCountBin(long bytes) { } return String.format("%.1f %cB", bytes / 1000.0, ci.current()); } - - } From 75871d522967920aea66681db4d86f492db7505f Mon Sep 17 00:00:00 2001 From: scezen Date: Tue, 13 Aug 2024 09:49:08 +0200 Subject: [PATCH 48/52] Cleanup --- .../ui/pages/OpenDataController.java | 2 + .../ui/services/OpenDataService.java | 7 +- .../resources/templates/opendata-isbn.html | 183 +++++++++++++- ui/src/main/resources/templates/opendata.html | 227 +++++++----------- 4 files changed, 264 insertions(+), 155 deletions(-) diff --git a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java index ffa4ed8e1..9d2bf8f7f 100644 --- a/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java +++ b/ui/src/main/java/org/open4goods/ui/controllers/ui/pages/OpenDataController.java @@ -65,6 +65,8 @@ public List getMultipleExposedUrls() { public ModelAndView opendata(final HttpServletRequest request) { final ModelAndView ret = uiService.defaultModelAndView("opendata", request); ret.addObject("count", openDataService.totalItems()); + ret.addObject("countGTIN", openDataService.totalItemsGTIN()); + ret.addObject("countISBN", openDataService.totalItemsISBN()); ret.addObject("isbnLastUpdated", openDataService.isbnLastUpdate()); ret.addObject("isbnFileSize", openDataService.isbnFileSize()); ret.addObject("gtinLastUpdated", openDataService.gtinLastUpdate()); diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 79e500aa4..71f26e222 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -71,7 +71,6 @@ public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiCo this.aggregatedDataRepository = aggregatedDataRepository; this.uiConfig = uiConfig; this.openDataConfig = openDataConfig; - generateOpendata(); } /** @@ -236,7 +235,7 @@ private String[] toGtinEntry(Product data) { line[8] = String.valueOf(data.bestPrice().getPrice()); // "min_price" line[9] = String.valueOf(data.bestPrice().getCompensation()); // "min_price_compensation" line[10] = data.bestPrice().getCurrency().toString(); // "currency" - line[12] = ""; // TODO: Construct the URL + line[12] = ""; // TODO: uiConfig.getBaseUrl(Locale.FRANCE) + data.getNames().getName(); } line[11] = StringUtils.join(data.getDatasourceCategories(), " ; "); // "categories" @@ -263,7 +262,7 @@ private String[] toIsbnEntry(Product data) { line[8] = String.valueOf(data.bestPrice().getPrice()); // "min_price" line[9] = String.valueOf(data.bestPrice().getCompensation()); // "min_price_compensation" line[10] = data.bestPrice().getCurrency().toString(); // "currency" - line[12] = ""; // TODO: Construct the URL + line[12] = ""; // TODO: uiConfig.getBaseUrl(Locale.FRANCE) + data.getNames().getName(); } line[11] = StringUtils.join(data.getDatasourceCategories(), " ; "); // "categories" @@ -282,7 +281,7 @@ private String[] toIsbnEntry(Product data) { /** * Retrieves a specific attribute from the product data. - * TODO : Review when all attributes acessible by map + * TODO : Review when all attributes accessible by map */ private String getAttribute(Product data, String key) { diff --git a/ui/src/main/resources/templates/opendata-isbn.html b/ui/src/main/resources/templates/opendata-isbn.html index 76156d78b..b997a2d7f 100644 --- a/ui/src/main/resources/templates/opendata-isbn.html +++ b/ui/src/main/resources/templates/opendata-isbn.html @@ -1,12 +1,183 @@ - + + - ISBN Open Data + + Base de données ISBN en libre accès | Nudger + + + + + + + + + + + + -

ISBN Open Data

-

Last Updated:

-

File Size:

-Download ISBN Data + + +
+
+
+
+
+

Open Data : Base de données ISBN en libre accès

+
+
+
+
+
+
+
+
+
+
+
+ code barre en opendata +
+
+

Libérez les données ! Accédez à notre base de données ISBN. Ces données sont mises à jour une fois par semaine et proviennent de notre agrégation de contenu autour des catalogues d'affiliation.

+
+
+

Conditions d'utilisation

+
    +
  • Ces données sont délivrées sous licence ouverte, et sont donc librement utilisables, sans restrictions d'usages.
    • +
  • Vous devez indiquer la provenance des données, en mentionnant la page produit quand disponible, ou en ajoutant un lien vers la page d'accueil de nudger.fr.
    • +
+
+
+
+
+

Obtenir les données

+
mise à jourMise à jour
+ + + + + + + + + + + + + + +
mise à jour
Taille du fichier
Nombre de produits
+ +
+ + + + + Télécharger + +
+ + + + +

Format des données

+ +
+
+ Colonnes +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
barcodeLe code barre du produit, qui est un ISBN
brandMarque de l'éditeur
modelRéférence de l'ouvrage
last_updatedDernière mise à jour de ce produit, au format epoch en millisecondes
gs1_countryPays de l'organisme GS1 qui a délivré le code barre pour ce produit
gtinTypeType du code barre. Pour ISBN, cela sera toujours ISBN_13
offers_countNombre d'offres commerciales pour ce produit
min_pricePrix le plus faible pour ce produit
min_price_compensationCompensation carbone reversée, correspondant au prix le plus faible pour ce produit
currencyDevise du prix le plus faible et de la compensation carbone associée
categoriesCatégories dans lesquelles ce produit a été rencontré
urlURL de la fiche produit, uniquement si des offres commerciales sont disponibles
+
+ + + + + + + + + + + + + + + + + diff --git a/ui/src/main/resources/templates/opendata.html b/ui/src/main/resources/templates/opendata.html index 41d980ca7..476d12d8f 100644 --- a/ui/src/main/resources/templates/opendata.html +++ b/ui/src/main/resources/templates/opendata.html @@ -46,161 +46,98 @@

Open Data : Base de données GTIN, EAN et ISBN en lib - - -
-
-
-
-
- -
-
- code barre en opendata -
-
-

Liberez, la donnée ! Nous mettons à disposition une extraction complète de notre base de données, celle-ci est notament utile pour le grand nombre d'articles identifiés par leurs codes barres (gtin / ean13 / UPC / ISBN). Ces données sont mises à jour une fois par semaine, et proviennent de l'aggrégation de contenu que nous réalisons autour des catalogues d'affiliations.

-
-
- -

Conditions d'utilisation

+ + +
+
+
+
+
+
+
+ code barre en opendata +
+
+

Libérez la donnée ! Nous mettons à disposition une extraction complète de notre base de données, celle-ci est notamment utile pour le grand nombre d'articles identifiés par leurs codes barres (GTIN / EAN13 / UPC / ISBN). Ces données sont mises à jour une fois par semaine, et proviennent de l'agrégation de contenu que nous réalisons autour des catalogues d'affiliations.

+
+
+ +

Conditions d'utilisation

  • Ces données sont délivrées sous licence ouverte, et sont donc librement utilisables, sans restrictions d'usages.
  • Vous devez par contre indiquer la provenance de ces données, en mentionnant la page produit quand elle est disponible, ou avec un lien vers la page d'accueil de nudger.fr quand la page produit n'est pas disponible.
    • -
- -
-
-
-
-

Obtenir les données

- - - - - - - - - - - - - - - - - -
Mise à jour
Taille du fichier
Nombre de produits
- + + +
+
+
+
+

 

+ + + + + + + + + + + + +
Mise à jour
Taille du fichier
Nombre de produits
-
- - - - - - Télécharger + +
+

Dataset GTIN

+ + + + + + + + + + + + +
+ + Voir les détails et télécharger + +
+ +
+

Dataset ISBN

+ + + + + + + + + + + + +
+ + Voir les détails et télécharger
-
- - - -

Format des données

- - - -
-
- Colonnes -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
barcodeLe code barre du produit, qui peut être un GTIN, EAN13 ou ISBN
brandMarque du produit
modelRéference constructeur du produit
last_updatedDernière mise à jour de ce produit, au format epoch en millisecondes
gs1_countryPays de l'organisme GS1 qui a délivré le code barre pour ce produit (hors ISBN)
gtinTypeType du code barre. Peut être ISBN_13, GTIN_13, GTIN_8, GTIN_12
offers_countNombre d'offres commerciales pour ce produit
min_pricePrix le plus faible pour ce produit
min_price_compensationCompensation carbone reversée, correspondant au prix le plus faible pour ce produit
currencyDevise du prix le plus faible et de la compensation carbone associée
categoriesCatégories dans lesquelles ce produit a été rencontré
urlURL de la fiche produit, uniquement si des offres commerciales sont disponibles
- - - - - -
-
-
-
-
- - - +
+
+
+
+
+ + From 6932205c69acd5f60fa57dd79e477a4daca236b7 Mon Sep 17 00:00:00 2001 From: scezen Date: Tue, 13 Aug 2024 10:00:20 +0200 Subject: [PATCH 49/52] Add health check and timing metrics --- .../ui/services/OpenDataService.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index 71f26e222..a44179ccd 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -12,6 +12,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import io.micrometer.core.annotation.Timed; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; @@ -27,6 +28,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.HealthIndicator; import org.springframework.cache.annotation.Cacheable; import org.springframework.scheduling.annotation.Scheduled; @@ -38,7 +41,7 @@ * Handles the creation of GTIN and ISBN datasets, their compression into ZIP files, * and the management of download limits and rates. */ -public class OpenDataService { +public class OpenDataService implements HealthIndicator { private static final Logger LOGGER = LoggerFactory.getLogger(OpenDataService.class); @@ -73,6 +76,21 @@ public OpenDataService(ProductRepository aggregatedDataRepository, UiConfig uiCo this.openDataConfig = openDataConfig; } + /** + * This method checks the health status of the OpenDataService. + * It verifies if the required ZIP files (ISBN and GTIN) exist. + */ + @Override + public Health health() { + if (!uiConfig.isbnZipFile().exists()) { + return Health.down().withDetail("ISBN Zip File", "Le fichier ZIP ISBN est manquant.").build(); + } + if (!uiConfig.gtinZipFile().exists()) { + return Health.down().withDetail("GTIN Zip File", "Le fichier ZIP GTIN est manquant.").build(); + } + return Health.up().withDetail("OpenDataService", "Tous les fichiers ZIP sont présents.").build(); + } + /** * Provides a limited bandwidth stream for downloading the opendata file. * Enforces a limit on the number of concurrent downloads. @@ -109,6 +127,7 @@ public void close() throws IOException { * TODO : Schedule in conf */ @Scheduled(initialDelay = 1000L * 3600, fixedDelay = 1000L * 3600 * 24 * 7) + @Timed(value = "OpenDataService.generateOpendata.time", description = "Time taken to generate the OpenData ZIP files", extraTags = {"service", "OpenDataService"}) public void generateOpendata() { if (exportRunning.getAndSet(true)) { From 50ab3643907a6c0e832b79d7dacb5fb0e9accc68 Mon Sep 17 00:00:00 2001 From: goulven Date: Tue, 13 Aug 2024 10:22:10 +0200 Subject: [PATCH 50/52] todo --- ui/src/main/java/org/open4goods/ui/services/OpenDataService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java index a44179ccd..cd1ed75e6 100644 --- a/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java +++ b/ui/src/main/java/org/open4goods/ui/services/OpenDataService.java @@ -286,6 +286,7 @@ private String[] toIsbnEntry(Product data) { line[11] = StringUtils.join(data.getDatasourceCategories(), " ; "); // "categories" + //TODO : Asconsts line[13] = getAttribute(data, "EDITEUR"); line[14] = getAttribute(data, "FORMAT"); line[15] = getAttribute(data, "NB DE PAGES"); From ce5a8741c26986e8304967db0ab5a3116cc40f3d Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 13 Sep 2024 18:39:23 +0200 Subject: [PATCH 51/52] Frontend --- .../resources/templates/opendata-gtin.html | 11 +--- .../resources/templates/opendata-isbn.html | 63 ++++++++++--------- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/ui/src/main/resources/templates/opendata-gtin.html b/ui/src/main/resources/templates/opendata-gtin.html index a1ee654a4..58639708e 100644 --- a/ui/src/main/resources/templates/opendata-gtin.html +++ b/ui/src/main/resources/templates/opendata-gtin.html @@ -50,16 +50,7 @@

Qu'est-ce qu'un GTIN ?

  •    Utilisé internationalement
  •    Facilite la gestion des stocks
    • -
  •    Optimise la logistique
    • -
-

L'importance des GTIN pour l'écologie

-

- Les GTIN jouent un rôle clé dans la traçabilité des produits, ce qui est essentiel pour une gestion écologique efficace. En permettant un suivi précis des produits tout au long de la chaîne d'approvisionnement, les GTIN aident à réduire le gaspillage et à optimiser l'utilisation des ressources. -

-
    -
  •    Réduction du gaspillage
    • -
  •    Optimisation des ressources
    • -
  •    Support à l'économie circulaire
    • +
  •   Optimise la logistique

Conditions d'utilisation

    diff --git a/ui/src/main/resources/templates/opendata-isbn.html b/ui/src/main/resources/templates/opendata-isbn.html index b997a2d7f..7f01191a5 100644 --- a/ui/src/main/resources/templates/opendata-isbn.html +++ b/ui/src/main/resources/templates/opendata-isbn.html @@ -4,16 +4,16 @@ Base de données ISBN en libre accès | Nudger - + - + - + @@ -40,9 +40,18 @@

    Open Data : Base de données ISBN en libre accès

    -

    Libérez les données ! Accédez à notre base de données ISBN. Ces données sont mises à jour une fois par semaine et proviennent de notre agrégation de contenu autour des catalogues d'affiliation.

    +

    Accédez à notre base de données ISBN. Ces données sont mises à jour une fois par semaine et proviennent de notre agrégation de contenu autour des catalogues d'affiliation.

    +

    Qu'est-ce qu'un ISBN ?

    +

    + L'ISBN (International Standard Book Number) est un identifiant unique pour les livres et autres publications. Il est essentiel dans le monde de l'édition pour identifier, gérer, et suivre les publications à travers les librairies, éditeurs et distributeurs. +

    +
      +
    •    Utilisé mondialement dans l'édition
      • +
    •    Identification unique pour les publications
      • +
    •   Facilite la distribution et la gestion des stocks
      • +

    Conditions d'utilisation

    • Ces données sont délivrées sous licence ouverte, et sont donc librement utilisables, sans restrictions d'usages.
      • @@ -56,15 +65,15 @@

      Obtenir les données

      - - + + - + @@ -106,48 +115,40 @@

      Format des données

      mise à jourMise à jour
      Taille du fichier
      Nombre de produitsNombre de publications
      - - + + - - + + - - + + - - + + - - + + - - + + - + - - - - - + - - - - - + @@ -180,4 +181,4 @@

      Format des données

      - + \ No newline at end of file From 038d6f27de6c11050231343240b0fcb332356453 Mon Sep 17 00:00:00 2001 From: scezen Date: Fri, 13 Sep 2024 18:58:54 +0200 Subject: [PATCH 52/52] Minor tweaks, formatting --- .../resources/templates/opendata-gtin.html | 10 ----- .../resources/templates/opendata-isbn.html | 42 ++++++++++++++----- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/ui/src/main/resources/templates/opendata-gtin.html b/ui/src/main/resources/templates/opendata-gtin.html index 58639708e..71fbcdf08 100644 --- a/ui/src/main/resources/templates/opendata-gtin.html +++ b/ui/src/main/resources/templates/opendata-gtin.html @@ -90,16 +90,6 @@

      Obtenir les données

      -

      Format des données

      -

      Format des données

       + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      barcodeLe code barre du produit, qui est un ISBNisbnLe code ISBN de la publication
      brandMarque de l'éditeurauthorAuteur de la publication
      modelRéférence de l'ouvragetitleTitre de la publication
      last_updatedDernière mise à jour de ce produit, au format epoch en millisecondespublisherÉditeur de la publication
      gs1_countryPays de l'organisme GS1 qui a délivré le code barre pour ce produitlast_updatedDernière mise à jour de cette publication, au format epoch en millisecondes
      gtinTypeType du code barre. Pour ISBN, cela sera toujours ISBN_13categoriesCatégories dans lesquelles cette publication a été rencontrée
      offers_countNombre d'offres commerciales pour ce produitNombre d'offres commerciales pour cette publication
      min_pricePrix le plus faible pour ce produit
      min_price_compensationCompensation carbone reversée, correspondant au prix le plus faible pour ce produitPrix le plus faible pour cette publication
      currencyDevise du prix le plus faible et de la compensation carbone associée
      categoriesCatégories dans lesquelles ce produit a été rencontréDevise du prix le plus faible
      url url URL de la fiche produit, uniquement si des offres commerciales sont disponibles
      editeurÉditeur de la publication
      formatFormat de la publication (ex. broché, numérique)
      nb de pagesNombre de pages de la publication
      classification decitre 1Classification principale de Decitre pour cette publication
      classification decitre 2Deuxième niveau de classification de Decitre
      classification decitre 3Troisième niveau de classification de Decitre
      souscategorieSous-catégorie principale pour cette publication
      souscategorie2Deuxième sous-catégorie pour cette publication