diff --git a/backend/conf/config.sample.properties b/backend/conf/config.sample.properties index fa2d05cd49..a3d13d0fea 100644 --- a/backend/conf/config.sample.properties +++ b/backend/conf/config.sample.properties @@ -44,6 +44,10 @@ backend.id = BackendServer1 # By default running executions are restarted, if false, executions are failed by backend at startup # backend.startup.restart.running = true +# Automatic delete of old executions limit in days - delete all executions older than the defined count of days +# By default set to -1 which means no executions are automatically cleaned up +# backend.execution.cleanup.days.limit = -1 + # Connection configuration setting for relational database # for mysql { database.sql.driver = com.mysql.jdbc.Driver diff --git a/backend/src/main/java/cz/cuni/mff/xrg/odcs/backend/execution/pipeline/impl/CleanUp.java b/backend/src/main/java/cz/cuni/mff/xrg/odcs/backend/execution/pipeline/impl/CleanUp.java index 7fb79c0176..a1d61f735f 100644 --- a/backend/src/main/java/cz/cuni/mff/xrg/odcs/backend/execution/pipeline/impl/CleanUp.java +++ b/backend/src/main/java/cz/cuni/mff/xrg/odcs/backend/execution/pipeline/impl/CleanUp.java @@ -17,10 +17,8 @@ package cz.cuni.mff.xrg.odcs.backend.execution.pipeline.impl; import java.io.File; -import java.io.IOException; import java.util.Map; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -100,7 +98,7 @@ public boolean postAction(PipelineExecution execution, } if (execution.isDebugging()) { // rdfDataUnitFactory.release(execution.getContext().generatePipelineId()); - + try { repositoryManager.release(execution.getContext().getExecutionId()); } catch (RDFException ex) { @@ -126,26 +124,26 @@ public boolean postAction(PipelineExecution execution, if (!execution.isDebugging()) { // delete working directory the sub directories should be already deleted by DPU's. try { - delete(resourceManager.getExecutionDir(execution)); - } catch (MissingResourceException ex ){ + CleanupUtils.deleteDirectory(this.resourceManager.getExecutionDir(execution)); + } catch (MissingResourceException ex) { LOG.warn("Can't delete directory.", ex); } } // delete result, storage if empty try { - deleteIfEmpty(resourceManager.getExecutionWorkingDir(execution)); - } catch (MissingResourceException ex ){ + CleanupUtils.deleteDirectoryIfEmpty(this.resourceManager.getExecutionWorkingDir(execution)); + } catch (MissingResourceException ex) { LOG.warn("Can't delete directory.", ex); } try { - deleteIfEmpty(resourceManager.getExecutionStorageDir(execution)); - } catch (MissingResourceException ex ){ + CleanupUtils.deleteDirectoryIfEmpty(this.resourceManager.getExecutionStorageDir(execution)); + } catch (MissingResourceException ex) { LOG.warn("Can't delete directory.", ex); } try { - deleteIfEmpty(resourceManager.getExecutionDir(execution)); - } catch (MissingResourceException ex ){ + CleanupUtils.deleteDirectoryIfEmpty(this.resourceManager.getExecutionDir(execution)); + } catch (MissingResourceException ex) { LOG.warn("Can't delete directory.", ex); } @@ -153,50 +151,4 @@ public boolean postAction(PipelineExecution execution, return true; } - /** - * Try to delete directory in execution directory. If error occur then is - * logged but otherwise ignored. - * - * @param toDelete - */ - private void delete(File toDelete) { - LOG.debug("Deleting: {}", toDelete.toString()); - - try { - FileUtils.deleteDirectory(toDelete); - } catch (IOException e) { - LOG.warn("Can't delete directory after execution", e); - } - } - - /** - * Delete directory if it's empty. - * - * @param toDelete - */ - private void deleteIfEmpty(File toDelete) { - if (!toDelete.exists()) { - // file does not exist - return; - } - - LOG.debug("Deleting: {}", toDelete.toString()); - - if (!toDelete.isDirectory()) { - LOG.warn("Directory to delete is file: {}", toDelete.toString()); - return; - } - - // check if empty - if (toDelete.list().length == 0) { - // empty - try { - FileUtils.deleteDirectory(toDelete); - } catch (IOException e) { - LOG.warn("Can't delete directory after execution", e); - } - } - - } - } diff --git a/backend/src/main/java/cz/cuni/mff/xrg/odcs/backend/execution/pipeline/impl/CleanupThread.java b/backend/src/main/java/cz/cuni/mff/xrg/odcs/backend/execution/pipeline/impl/CleanupThread.java new file mode 100644 index 0000000000..b7ce9436fe --- /dev/null +++ b/backend/src/main/java/cz/cuni/mff/xrg/odcs/backend/execution/pipeline/impl/CleanupThread.java @@ -0,0 +1,157 @@ +/** + * This file is part of UnifiedViews. + * + * UnifiedViews is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * UnifiedViews is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with UnifiedViews. If not, see . + */ +package cz.cuni.mff.xrg.odcs.backend.execution.pipeline.impl; + +import java.io.File; +import java.util.Calendar; +import java.util.List; + +import javax.annotation.PostConstruct; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import cz.cuni.mff.xrg.odcs.commons.app.conf.AppConfig; +import cz.cuni.mff.xrg.odcs.commons.app.conf.ConfigProperty; +import cz.cuni.mff.xrg.odcs.commons.app.facade.LogFacade; +import cz.cuni.mff.xrg.odcs.commons.app.facade.PipelineFacade; +import cz.cuni.mff.xrg.odcs.commons.app.pipeline.PipelineExecution; +import cz.cuni.mff.xrg.odcs.commons.app.pipeline.PipelineExecutionStatus; +import cz.cuni.mff.xrg.odcs.commons.app.resource.MissingResourceException; +import cz.cuni.mff.xrg.odcs.commons.app.resource.ResourceManager; + +/** + * This class periodically (every 24h) checks database for old executions executed by this backend + * and deletes all executions older than the defined count of days + * Temporary data for all failed and canceled non-debugged executions are also cleaned up during this check (regardless the execution age) + * as these data cannot be accessed from UV in any way and thus just take up place on disk + */ +@Component +public class CleanupThread { + + private static Logger LOG = LoggerFactory.getLogger(CleanupThread.class); + + @Autowired + private ResourceManager resourceManager; + + private int executionsDaysLimitForCleanup = -1; + + @Autowired + private AppConfig appConfig; + + @Autowired + private PipelineFacade pipelineFacade; + + @Autowired + private LogFacade logFacade; + + private String backendId; + + @PostConstruct + public void init() { + this.backendId = this.appConfig.getString(ConfigProperty.BACKEND_ID); + if (this.appConfig.contains(ConfigProperty.BACKEND_EXECUTION_CLEANUP_DAYS_LIMIT)) { + this.executionsDaysLimitForCleanup = this.appConfig.getInteger(ConfigProperty.BACKEND_EXECUTION_CLEANUP_DAYS_LIMIT); + } + } + + /** + * Periodically (every 24 hours) cleanup the executions + */ + @Async + @Scheduled(fixedRate = 24 * 60 * 60 * 1000) + protected void cleanupExecutions() { + LOG.info("Going to cleanup executions"); + if (this.executionsDaysLimitForCleanup != -1) { + LOG.info("Deleting all executions older than {} days", this.executionsDaysLimitForCleanup); + deleteExecutions(); + } + + LOG.info("Deleting temp data for all failed/cancelled executions"); + deleteTempDataForFailedExecutions(); + LOG.info("Executions cleanup successfully finished"); + } + + /** + * Delete all finished executions older than the defined count of days and all its data and files + * Deletes execution from DB along with logs, events and deletes execution files from disk + * Note: This code duplicates the code also defined in {@link cz.cuni.mff.xrg.odcs.frontend.gui.views.Settings.PipelineExecutionDeleterThread} (frontend) + */ + private void deleteExecutions() { + List finishedPipelineExecutions = this.pipelineFacade.getAllExecutions(PipelineExecutionStatus.CANCELLED, this.backendId); + finishedPipelineExecutions.addAll(this.pipelineFacade.getAllExecutions(PipelineExecutionStatus.FAILED, this.backendId)); + finishedPipelineExecutions.addAll(this.pipelineFacade.getAllExecutions(PipelineExecutionStatus.FINISHED_SUCCESS, this.backendId)); + finishedPipelineExecutions.addAll(this.pipelineFacade.getAllExecutions(PipelineExecutionStatus.FINISHED_WARNING, this.backendId)); + + int recordsDeleted = 0; + int recordsDeleteFailed = 0; + + Calendar now = Calendar.getInstance(); + now.add(java.util.Calendar.HOUR, -24 * this.executionsDaysLimitForCleanup); + for (PipelineExecution fex : finishedPipelineExecutions) { + Calendar executionEnd = Calendar.getInstance(); + executionEnd.setTime(fex.getEnd()); + try { + if (executionEnd.before(now)) { + try { + final File executionDir = this.resourceManager.getExecutionDir(fex); + CleanupUtils.deleteDirectory(executionDir); + } catch (MissingResourceException ex) { + LOG.warn("No resources to delete for Pipeline execution id: {}", fex.getId(), ex); + } + this.logFacade.deleteLogs(fex); + this.pipelineFacade.delete(fex); + recordsDeleted++; + } + } catch (Exception e) { + LOG.error("Failed to cleanup execution {}", fex.getId(), e); + recordsDeleteFailed++; + } + } + LOG.info("Executions cleanup finished, {} executions successfully cleaned, failed to clean {} executions", recordsDeleted, recordsDeleteFailed); + } + + /** + * Delete temp execution files for non debugging failed and canceled executions + * Note: Check whether this code is needed. Because there is CleanUp PostExecutor which, for every finished pipeline + * (even those which fail or are cancelled) cleans up the working directory. + */ + private void deleteTempDataForFailedExecutions() { + List failedExecutions = this.pipelineFacade.getAllExecutions(PipelineExecutionStatus.FAILED, this.backendId); + failedExecutions.addAll(this.pipelineFacade.getAllExecutions(PipelineExecutionStatus.CANCELLED, this.backendId)); + int cleanedExecutions = 0; + for (PipelineExecution failed : failedExecutions) { + if (!failed.isDebugging()) { + try { + final File executionDir = this.resourceManager.getExecutionDir(failed); + CleanupUtils.deleteDirectory(executionDir); + cleanedExecutions++; + } catch (MissingResourceException e) { + LOG.info("No resources to delete for Pipeline execution id: {}", failed.getId(), e); + } catch (Exception e) { + LOG.error("Failed to delete temp data for execution {}", failed.getId(), e); + } + } + } + LOG.info("Deleted temp execution data for {} failed/cancelled executions", cleanedExecutions); + } + +} diff --git a/backend/src/main/java/cz/cuni/mff/xrg/odcs/backend/execution/pipeline/impl/CleanupUtils.java b/backend/src/main/java/cz/cuni/mff/xrg/odcs/backend/execution/pipeline/impl/CleanupUtils.java new file mode 100644 index 0000000000..65165b4f10 --- /dev/null +++ b/backend/src/main/java/cz/cuni/mff/xrg/odcs/backend/execution/pipeline/impl/CleanupUtils.java @@ -0,0 +1,76 @@ +/** + * This file is part of UnifiedViews. + * + * UnifiedViews is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * UnifiedViews is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with UnifiedViews. If not, see . + */ +package cz.cuni.mff.xrg.odcs.backend.execution.pipeline.impl; + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CleanupUtils { + + private static Logger LOG = LoggerFactory.getLogger(CleanupUtils.class); + + /** + * Try to delete directory in execution directory. If error occur then is + * logged but otherwise ignored. + * + * @param toDelete + */ + public static void deleteDirectory(File toDelete) { + LOG.debug("Deleting: {}", toDelete.toString()); + + try { + FileUtils.deleteDirectory(toDelete); + } catch (IOException e) { + LOG.warn("Can't delete directory after execution", e); + } + } + + /** + * Delete directory if it's empty. + * + * @param toDelete + */ + public static void deleteDirectoryIfEmpty(File toDelete) { + if (!toDelete.exists()) { + // file does not exist + return; + } + + LOG.debug("Deleting: {}", toDelete.toString()); + + if (!toDelete.isDirectory()) { + LOG.warn("Directory to delete is file: {}", toDelete.toString()); + return; + } + + // check if empty + if (toDelete.list().length == 0) { + // empty + try { + FileUtils.deleteDirectory(toDelete); + } catch (IOException e) { + LOG.warn("Can't delete directory after execution", e); + } + } + + } + +} diff --git a/commons-app/src/main/java/cz/cuni/mff/xrg/odcs/commons/app/conf/ConfigProperty.java b/commons-app/src/main/java/cz/cuni/mff/xrg/odcs/commons/app/conf/ConfigProperty.java index 6f926c1b29..bc07600ff7 100644 --- a/commons-app/src/main/java/cz/cuni/mff/xrg/odcs/commons/app/conf/ConfigProperty.java +++ b/commons-app/src/main/java/cz/cuni/mff/xrg/odcs/commons/app/conf/ConfigProperty.java @@ -37,6 +37,7 @@ public enum ConfigProperty { BACKEND_ID("backend.id"), BACKEND_STARTUP_RESTART_RUNNING("backend.startup.restart.running"), LOCALE("locale"), + BACKEND_EXECUTION_CLEANUP_DAYS_LIMIT("backend.execution.cleanup.days.limit"), EXECUTION_LOG_HISTORY("exec.log.history"), EXECUTION_LOG_SIZE_MAX("exec.log.msg.maxSize"), diff --git a/frontend/src/main/java/cz/cuni/mff/xrg/odcs/frontend/gui/views/Settings.java b/frontend/src/main/java/cz/cuni/mff/xrg/odcs/frontend/gui/views/Settings.java index fb06104751..fb775de94b 100644 --- a/frontend/src/main/java/cz/cuni/mff/xrg/odcs/frontend/gui/views/Settings.java +++ b/frontend/src/main/java/cz/cuni/mff/xrg/odcs/frontend/gui/views/Settings.java @@ -86,7 +86,7 @@ * GUI for Settings page which opens from the main menu. For User role it * contains Email notifications form. For Administrator role it contains extra * functionality: Users list, Prune execution records, Release locked pipelines - * + * * @author Maria Kukhar */ @org.springframework.stereotype.Component @@ -258,12 +258,12 @@ private GridLayout buildMainLayout() { usersLayout.setStyleName("settings"); //layout for Namespace Prefixes -// prefixesLayout = new VerticalLayout(); -// prefixesLayout.setImmediate(true); -// prefixesLayout.setWidth("100%"); -// prefixesLayout.setHeight("100%"); -// prefixesLayout = prefixesList.buildNamespacePrefixesLayout(); -// prefixesLayout.setStyleName("settings"); + // prefixesLayout = new VerticalLayout(); + // prefixesLayout.setImmediate(true); + // prefixesLayout.setWidth("100%"); + // prefixesLayout.setHeight("100%"); + // prefixesLayout = prefixesList.buildNamespacePrefixesLayout(); + // prefixesLayout.setStyleName("settings"); //My account tab accountButton = new NativeButton(Messages.getString("Settings.my.account")); @@ -343,28 +343,28 @@ public void buttonClick(ClickEvent event) { tabsLayout.setComponentAlignment(usersButton, Alignment.TOP_RIGHT); //Namespace prefixes tab -// prefixesButton = new NativeButton("Namespace Prefixes"); -// prefixesButton.setHeight("40px"); -// prefixesButton.setWidth("170px"); -// prefixesButton.setStyleName("multiline"); -// prefixesButton.setVisible(loggedUser.getRoles().contains(Role.ROLE_ADMIN)); -// prefixesButton.addClickListener(new ClickListener() { -// private static final long serialVersionUID = 1L; -// -// @Override -// public void buttonClick(ClickEvent event) { -// if (shownTab.equals(accountButton)) { -// myAccountSaveConfirmation(prefixesButton, prefixesLayout); -// } else { -// if (shownTab.equals(notificationsButton)) { -// notificationSaveConfirmation(prefixesButton, -// prefixesLayout); -// } else { -// buttonPush(prefixesButton, prefixesLayout); -// } -// } -// } -// }); + // prefixesButton = new NativeButton("Namespace Prefixes"); + // prefixesButton.setHeight("40px"); + // prefixesButton.setWidth("170px"); + // prefixesButton.setStyleName("multiline"); + // prefixesButton.setVisible(loggedUser.getRoles().contains(Role.ROLE_ADMIN)); + // prefixesButton.addClickListener(new ClickListener() { + // private static final long serialVersionUID = 1L; + // + // @Override + // public void buttonClick(ClickEvent event) { + // if (shownTab.equals(accountButton)) { + // myAccountSaveConfirmation(prefixesButton, prefixesLayout); + // } else { + // if (shownTab.equals(notificationsButton)) { + // notificationSaveConfirmation(prefixesButton, + // prefixesLayout); + // } else { + // buttonPush(prefixesButton, prefixesLayout); + // } + // } + // } + // }); //tabsLayout.addComponent(prefixesButton); //tabsLayout.setComponentAlignment(prefixesButton, Alignment.TOP_RIGHT); @@ -632,7 +632,7 @@ private void refreshRuntimeProperties() { /** * Validates and return the TextField.value - * + * * @param layout * GridLayout * @param column @@ -659,7 +659,7 @@ private String validateAndGetValue(Component layout, int column) throws InvalidV /** * Building Schedule notifications layout. Appear after pushing Schedule * notifications tab - * + * * @return notificationsLayout Layout with components of Schedule * notifications. */ @@ -686,7 +686,7 @@ private VerticalLayout buildNotificationsLayout() { /** * Building My account layout. Appear after pushing My account tab - * + * * @return accountLayout Layout with components of My account. */ private VerticalLayout buildMyAccountLayout() { @@ -748,7 +748,7 @@ public void textChange(FieldEvents.TextChangeEvent event) { /** * Building layout with button Save for saving My account tab - * + * * @return buttonBar Layout with button */ private HorizontalLayout buildButtonMyAccountBar() { @@ -782,7 +782,7 @@ public void buttonClick(ClickEvent event) { /** * Building layout with button Save for saving notifications - * + * * @return buttonBar Layout with button */ private HorizontalLayout buildButtonNotificationBar() { @@ -817,7 +817,7 @@ public void buttonClick(ClickEvent event) { /** * Showing active tab. - * + * * @param pressedButton * Tab that was pressed. * @param layoutShow @@ -905,7 +905,7 @@ private boolean saveEmailNotifications() { * tab and push anoter tab. User can save changes or discard. After that * will be shown another selected tab. If there was no changes, a * confirmation window will not be shown. - * + * * @param pressedButton * New tab that was push. * @param layoutShow @@ -946,7 +946,7 @@ public void onClose(ConfirmDialog cd) { * notifications tab and push another tab. User can save changes or discard. * After that will be shown another selected tab. If there was no changes, a * confirmation window will not be shown. - * + * * @param pressedButton * New tab that was push. * @param layoutShow @@ -1079,7 +1079,7 @@ private String emailValidationText() { /** * Check for permission. - * + * * @param type * Required permission. * @return If the user has given permission @@ -1194,6 +1194,9 @@ class PipelineExecutionDeleterThread implements Runnable { } @Override + /** + * Note: This code duplicates the code also defined in {@link cz.cuni.mff.xrg.odcs.backend.execution.pipeline.impl.CleanupThread} (backend) + */ public void run() { try { SecurityContextHolder.getContext().setAuthentication(authentication);