Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add azure batch pool virtualNetwork option #3723

Merged
merged 2 commits into from
Mar 6, 2023
Merged

Add azure batch pool virtualNetwork option #3723

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b9f00bc
Add azure batch pool virtualNetwork option
bentsherman Mar 6, 2023
c473e15
Update docs/azure.rst [ci skip]
bentsherman Mar 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/azure.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,7 @@ azure.batch.pools.<name>.scaleInterval Specify the interval at which to
azure.batch.pools.<name>.schedulePolicy Specify the scheduling policy for the pool identified with ``<name>``. It can be either ``spread`` or ``pack`` (default: ``spread``).
azure.batch.pools.<name>.privileged Enable the task to run with elevated access. Ignored if `runAs` is set (default: ``false``).
azure.batch.pools.<name>.runAs Specify the username under which the task is run. The user must already exist on each node of the pool.
azure.batch.pools.<name>.virtualNetwork Specify the subnet ID of a virtual network in which to create the pool (requires version ``23.03.0-edge`` or later).
azure.registry.server Specify the container registry from which to pull the Docker images (default: ``docker.io``, requires ``[email protected]``).
azure.registry.userName Specify the username to connect to a private container registry (requires ``[email protected]``).
azure.registry.password Specify the password to connect to a private container registry (requires ``[email protected]``).
Expand Down
6 changes: 5 additions & 1 deletion plugins/nf-azure/src/main/nextflow/cloud/azure/batch/AzBatchService.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import com.microsoft.azure.batch.protocol.models.ContainerRegistry
import com.microsoft.azure.batch.protocol.models.ElevationLevel
import com.microsoft.azure.batch.protocol.models.ImageInformation
import com.microsoft.azure.batch.protocol.models.MountConfiguration
import com.microsoft.azure.batch.protocol.models.NetworkConfiguration
import com.microsoft.azure.batch.protocol.models.OutputFile
import com.microsoft.azure.batch.protocol.models.OutputFileBlobContainerDestination
import com.microsoft.azure.batch.protocol.models.OutputFileDestination
Expand Down Expand Up @@ -663,7 +664,6 @@ class AzBatchService implements Closeable {
.withCommandLine('bash -c "chmod +x azcopy && mkdir \$AZ_BATCH_NODE_SHARED_DIR/bin/ && cp azcopy \$AZ_BATCH_NODE_SHARED_DIR/bin/" ')
.withResourceFiles(resourceFiles)


final poolParams = new PoolAddParameter()
.withId(spec.poolId)
.withVirtualMachineConfiguration(poolVmConfig(spec.opts))
Expand All @@ -674,6 +674,10 @@ class AzBatchService implements Closeable {
.withTaskSlotsPerNode(spec.vmType.numberOfCores)
.withStartTask(poolStartTask)

// virtual network
if( spec.opts.virtualNetwork )
poolParams.withNetworkConfiguration( new NetworkConfiguration().withSubnetId(spec.opts.virtualNetwork) )

// scheduling policy
if( spec.opts.schedulePolicy ) {
final pol = ComputeNodeFillType.fromString(spec.opts.schedulePolicy)
Expand Down
4 changes: 4 additions & 0 deletions plugins/nf-azure/src/main/nextflow/cloud/azure/config/AzPoolOpts.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ class AzPoolOpts implements CacheFunnel {
String userName
String password

String virtualNetwork

AzPoolOpts() {
this(Collections.emptyMap())
}
Expand All @@ -86,6 +88,7 @@ class AzPoolOpts implements CacheFunnel {
this.registry = opts.registry
this.userName = opts.userName
this.password = opts.password
this.virtualNetwork = opts.virtualNetwork
}

@Override
Expand All @@ -104,6 +107,7 @@ class AzPoolOpts implements CacheFunnel {
hasher.putBoolean(autoScale)
hasher.putUnencodedChars(scaleFormula ?: '')
hasher.putUnencodedChars(schedulePolicy ?: '')
hasher.putUnencodedChars(virtualNetwork ?: '')
return hasher
}

Expand Down