Skip to content

Commit

Permalink
Vacuum Task for Transient repositories #10690
Browse files Browse the repository at this point in the history 
(cherry picked from commit d7f125d)
  • Loading branch information
@anatol-sialitski
anatol-sialitski committed Dec 10, 2024
1 parent 7a80d78 commit ef91a35
Show file tree
Hide file tree
Showing 3 changed files with 134 additions and 21 deletions.
35 changes: 28 additions & 7 deletions .../src/main/java/com/enonic/xp/repo/impl/vacuum/versiontable/VersionTableVacuumCommand.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package com.enonic.xp.repo.impl.vacuum.versiontable;

import java.time.Clock;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.HashSet;
import java.util.Set;

Expand Down Expand Up @@ -58,6 +60,8 @@ public class VersionTableVacuumCommand

private final Instant until;

private final Instant untilForTransientRepository;

private final VacuumListener listener;

private final int batchSize;
Expand All @@ -76,9 +80,12 @@ private VersionTableVacuumCommand( final Builder builder )
versionService = builder.versionService;
blobStore = builder.blobStore;
branchService = builder.branchService;
until = Instant.now().minusMillis( builder.params.getAgeThreshold() );
listener = builder.params.getListener();
batchSize = builder.params.getVersionsBatchSize();

final Instant now = builder.clock != null ? Instant.now( builder.clock ) : Instant.now();
until = now.minusMillis( builder.params.getAgeThreshold() );
untilForTransientRepository = now.minus( 1, ChronoUnit.MINUTES );
}

public static Builder create()
Expand Down Expand Up @@ -109,7 +116,11 @@ private void doProcessRepository( final Repository repository )

NodeVersionId lastVersionId = null;

NodeVersionQuery query = createQuery( lastVersionId );
final Instant ageThreshold = repository.isTransient() ? untilForTransientRepository : until;

LOG.debug( "Repo is transient: {}, ageThreshold: {}", repository.isTransient(), ageThreshold );

NodeVersionQuery query = createQuery( lastVersionId, ageThreshold );
NodeVersionQueryResult versionsResult = nodeService.findVersions( query );
long hits = versionsResult.getHits();

Expand All @@ -134,6 +145,8 @@ private void doProcessRepository( final Repository repository )
final boolean toDelete = processVersion( repository, version );
if ( toDelete )
{
LOG.debug( "Version's timestamp = '{}', nodeId = '{}', versionId = '{}'", version.getTimestamp(), version.getNodeId(),
version.getNodeVersionId() );
result.deleted();
versionService.delete( version.getNodeVersionId(), context );
nodeBlobToCheckSet.add( version.getNodeVersionKey().getNodeBlobKey() );
Expand All @@ -155,7 +168,7 @@ private void doProcessRepository( final Repository repository )
.filter( blobKey -> !isBlobKeyUsed( blobKey, VersionIndexPath.BINARY_BLOB_KEYS ) )
.forEach( blobKey -> removeNodeBlobRecord( repository.getId(), NodeConstants.BINARY_SEGMENT_LEVEL, blobKey ) );

query = createQuery( lastVersionId );
query = createQuery( lastVersionId, ageThreshold );
versionsResult = nodeService.findVersions( query );
hits = versionsResult.getHits();
}
Expand Down Expand Up @@ -185,10 +198,10 @@ private boolean processVersion( final Repository repository, final NodeVersionMe
switch ( findVersionsInBranches( repository, version ) )
{
case NO_VERSION_FOUND:
LOG.debug( "No version found in branch for [{}/ {}]", version.getNodeId(), version.getNodeVersionId() );
LOG.debug( "No version found in branch for [{}/ {}]", version.getNodeId(), version.getNodeVersionId() );
return true;
case OTHER_VERSION_FOUND:
LOG.debug( "Other version found in branch for [{}/ {}]", version.getNodeId(), version.getNodeVersionId() );
LOG.debug( "Other version found in branch for [{}/ {}]", version.getNodeId(), version.getNodeVersionId() );
return version.getNodeCommitId() == null;
default:
return false;
Expand Down Expand Up @@ -222,7 +235,7 @@ private BRANCH_CHECK_RESULT findVersionsInBranches( final Repository repository,
return nodeFound ? BRANCH_CHECK_RESULT.OTHER_VERSION_FOUND : BRANCH_CHECK_RESULT.NO_VERSION_FOUND;
}

private NodeVersionQuery createQuery( NodeVersionId lastVersionId )
private NodeVersionQuery createQuery( NodeVersionId lastVersionId, Instant ageThreshold )
{
final NodeVersionQuery.Builder builder = NodeVersionQuery.create();

Expand All @@ -236,7 +249,7 @@ private NodeVersionQuery createQuery( NodeVersionId lastVersionId )
}

final RangeFilter mustBeOlderThanFilter =
RangeFilter.create().fieldName( VersionIndexPath.TIMESTAMP.getPath() ).to( ValueFactory.newDateTime( until ) ).build();
RangeFilter.create().fieldName( VersionIndexPath.TIMESTAMP.getPath() ).to( ValueFactory.newDateTime( ageThreshold ) ).build();

return builder.addQueryFilter( mustBeOlderThanFilter )
.addOrderBy( FieldOrderExpr.create( VersionIndexPath.VERSION_ID, OrderExpr.Direction.ASC ) )
Expand All @@ -246,6 +259,8 @@ private NodeVersionQuery createQuery( NodeVersionId lastVersionId )

public static final class Builder
{
private Clock clock;

private NodeService nodeService;

private RepositoryService repositoryService;
Expand Down Expand Up @@ -298,6 +313,12 @@ public Builder branchService( final BranchService branchService )
return this;
}

public Builder clock( final Clock clock )
{
this.clock = clock;
return this;
}

public VersionTableVacuumCommand build()
{
return new VersionTableVacuumCommand( this );
Expand Down
40 changes: 26 additions & 14 deletions ...epo/src/main/java/com/enonic/xp/repo/impl/vacuum/versiontable/VersionTableVacuumTask.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.enonic.xp.repo.impl.vacuum.versiontable;

import java.time.Clock;

import org.osgi.service.component.annotations.Activate;
import org.osgi.service.component.annotations.Component;
import org.osgi.service.component.annotations.Reference;
Expand Down Expand Up @@ -31,35 +33,45 @@ public class VersionTableVacuumTask

private final BlobStore blobStore;

private Clock clock;

@Activate
public VersionTableVacuumTask( @Reference final NodeService nodeService, @Reference final RepositoryService repositoryService,
@Reference final VersionService versionService, @Reference final
BranchService branchService, @Reference final BlobStore blobStore )
@Reference final VersionService versionService, @Reference final BranchService branchService,
@Reference final BlobStore blobStore )
{
this.nodeService = nodeService;
this.repositoryService = repositoryService;
this.versionService = versionService;
this.blobStore = blobStore;
this.branchService = branchService;
this.clock = Clock.systemUTC();
}

public void setClock( final Clock clock )
{
this.clock = clock;
}

@Override
public VacuumTaskResult execute( final VacuumTaskParams params )
{
if (params.hasListener()) {
if ( params.hasListener() )
{
params.getListener().taskBegin( NAME, null );
}
return VersionTableVacuumCommand.create().
repositoryService( repositoryService ).
nodeService( nodeService ).
versionService( versionService ).
branchService( branchService ).
blobStore( blobStore ).
params( params ).
build().
execute().
taskName( NAME ).
build();
return VersionTableVacuumCommand.create()
.repositoryService( repositoryService )
.nodeService( nodeService )
.versionService( versionService )
.branchService( branchService )
.blobStore( blobStore )
.params( params )
.clock( clock )
.build()
.execute()
.taskName( NAME )
.build();
}

@Override
Expand Down
80 changes: 80 additions & 0 deletions ...src/test/java/com/enonic/xp/core/repo/vacuum/versiontable/VersionTableVacuumTaskTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,18 @@
package com.enonic.xp.core.repo.vacuum.versiontable;

import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneId;
import java.time.temporal.ChronoUnit;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import com.enonic.xp.content.ContentConstants;
import com.enonic.xp.context.Context;
import com.enonic.xp.context.ContextAccessor;
import com.enonic.xp.context.ContextBuilder;
import com.enonic.xp.core.AbstractNodeTest;
import com.enonic.xp.node.Node;
import com.enonic.xp.node.NodeId;
Expand All @@ -16,6 +23,14 @@
import com.enonic.xp.repo.impl.node.NodeHelper;
import com.enonic.xp.repo.impl.vacuum.VacuumTaskParams;
import com.enonic.xp.repo.impl.vacuum.versiontable.VersionTableVacuumTask;
import com.enonic.xp.repository.CreateRepositoryParams;
import com.enonic.xp.repository.RepositoryId;
import com.enonic.xp.security.PrincipalKey;
import com.enonic.xp.security.RoleKeys;
import com.enonic.xp.security.User;
import com.enonic.xp.security.acl.AccessControlEntry;
import com.enonic.xp.security.acl.AccessControlList;
import com.enonic.xp.security.auth.AuthenticationInfo;
import com.enonic.xp.vacuum.VacuumTaskResult;

import static org.junit.jupiter.api.Assertions.assertEquals;
Expand Down Expand Up @@ -139,6 +154,71 @@ void age_threshold()
assertVersions( node1.id(), 3 );
}

@Test
void testDeleteOnTransientRepository()
{
final Context oldContext = ContextAccessor.current();

try
{
final RepositoryId repositoryId = RepositoryId.from( "com.test.transient" + System.currentTimeMillis() );
final Context context = ContextBuilder.create()
.branch( ContentConstants.BRANCH_MASTER )
.repositoryId( repositoryId )
.authInfo( AuthenticationInfo.create()
.principals( RoleKeys.ADMIN )
.user( User.create().key( PrincipalKey.ofSuperUser() ).login( PrincipalKey.ofSuperUser().getId() ).build() )
.build() )
.build();

ContextAccessor.INSTANCE.set( context );

context.callWith( () -> {
createTransientRepo( repositoryId );

Instant initTime = Instant.now();
Clock clock = Clock.fixed( initTime, ZoneId.systemDefault() );

this.task.setClock( clock );

final Node node1 = createNode( NodePath.ROOT, "node1" );
updateNode( node1.id(), 1 );

refresh();
assertVersions( node1.id(), 2 );

Instant newTime = initTime.plus( 3, ChronoUnit.MINUTES );
this.task.setClock( Clock.fixed( newTime, ZoneId.systemDefault() ) );

final VacuumTaskResult result = NodeHelper.runAsAdmin( () -> this.task.execute( VacuumTaskParams.create().build() ) );

refresh();

assertEquals( 3, result.getProcessed() );
assertEquals( 1, result.getDeleted() );

assertVersions( node1.id(), 1 );

return null;
} );
}
finally
{
ContextAccessor.INSTANCE.set( oldContext );
}
}

private void createTransientRepo( final RepositoryId repositoryId )
{
final AccessControlList rootPermissions =
AccessControlList.of( AccessControlEntry.create().principal( TEST_DEFAULT_USER.getKey() ).allowAll().build() );

this.repositoryService.createRepository(
CreateRepositoryParams.create().repositoryId( repositoryId ).rootPermissions( rootPermissions ).transientFlag( true ).build() );

refresh();
}

private void assertVersions( final NodeId nodeId, final int versions )
{
final NodeVersionQueryResult result = this.nodeService.findVersions( NodeVersionQuery.create().
Expand Down

0 comments on commit ef91a35

Please sign in to comment.