fix: don't create two reports when saveProgress report exists

viasite · Apr 2, 2024 · ad1dcb3 · ad1dcb3
1 parent faa8be9
commit ad1dcb3
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 6 deletions.
diff --git a/src/actions/copyJsonToReports.js b/src/actions/copyJsonToReports.js
@@ -3,13 +3,14 @@ import { getJsonName, getUserDir } from '../utils.js';
 
 const defaultLocalDir = 'data/reports/';
 
-export default (jsonPath, uid = '', localDir = defaultLocalDir, timestamp) => {
+export default (jsonPath, uid = '', localDir = defaultLocalDir, timestamp, overwrite = true) => {
   const userDir = getUserDir(uid, localDir);
 
   // remove microseconds if available
   const jsonNameLong = getJsonName(jsonPath, false, timestamp);
   const jsonNameShort = getJsonName(jsonPath, true, timestamp);
-  const jsonName = fs.existsSync(userDir + '/' + jsonNameShort) ? jsonNameLong : jsonNameShort;
+  // TODO2: overwrite always true, may overwrite needed report, forgot why it was added
+  const jsonName = fs.existsSync(userDir + '/' + jsonNameShort) && !overwrite ? jsonNameLong : jsonNameShort;
 
   // copy json file
   const localPath = userDir + '/' + jsonName;

diff --git a/src/actions/saveAsJson.js b/src/actions/saveAsJson.js
@@ -49,7 +49,7 @@ export default async ({csvPath, jsonPath, lang, preset, defaultFilter, url, args
 
   // filter empty redirected items
   console.log("data.items before filter:", data.items.length);
-  data.items = filterItems(data.items); // TODO: uncomment
+  data.items = filterItems(data.items);
 
   // write
   const raw = JSON.stringify(data);

diff --git a/src/scrap-site.js b/src/scrap-site.js
@@ -36,7 +36,7 @@ let disconnectedLog = [];
 
 // resend messages while disconnected
 function sendDisconnected(socket) {
-  if (disconnectedLog.length == 0) return;
+  if (disconnectedLog.length === 0) return;
   const log = [...disconnectedLog];
   disconnectedLog = [];
   for (let item of log) {
@@ -791,7 +791,7 @@ async function scrapSite ({baseUrl, options = {}}) {
       startTime,
       partNum: options.partNum,
     })
-    const { jsonName, localPath } = copyJsonToReports(jsonPath, options.socket.uid, undefined, startTime);
+    const { jsonName, localPath } = copyJsonToReports(jsonPath, options.socket.uid, undefined, startTime, true);
 
     // send result json to socket
     socketSend(options.socket, 'result', {name: jsonName, isProgress: true, count: items.length});
@@ -1022,7 +1022,7 @@ async function scrapSite ({baseUrl, options = {}}) {
       }
 
       if (options.webService) {
-        const { jsonName, localPath } = copyJsonToReports(jsonPath, options.socket.uid, undefined, options.partialFirstStartTime);
+        const { jsonName, localPath } = copyJsonToReports(jsonPath, options.socket.uid, undefined, options.partialFirstStartTime, true);;
 
         // send result json to socket
         socketSend(options.socket, 'result', {name: jsonName, count: data.items.length});