Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
aac1f48
Initial checkin of incomplete azure deletion changeset
jgainerdewar Jun 27, 2025
684322c
Liquibase changeset
jgainerdewar Jun 30, 2025
568af50
Cosmetic change for case consistency
jgainerdewar Jul 1, 2025
079e7e0
Merge branch 'develop' into jd_AN-648_dbAzureCleanup
jgainerdewar Jul 1, 2025
521fb14
Split changeset into one per high-level resource
jgainerdewar Jul 2, 2025
d42ece3
Remove app v2 endpoints and app v2 service methods
jgainerdewar Jul 3, 2025
95c74e2
Remove runtime v2 routes and runtime v2 service
jgainerdewar Jul 3, 2025
048b5e5
Remove disk v2 routes and disk v2 service
jgainerdewar Jul 3, 2025
cbaafc9
Cleanup
jgainerdewar Jul 3, 2025
d540984
Remove machinery to perform Azure app install and delete
jgainerdewar Jul 3, 2025
b37b8eb
Remove azure-only AppTypes and associated config
jgainerdewar Jul 3, 2025
65cba35
Remove app DAOs
jgainerdewar Jul 3, 2025
03f731e
Remove AzurePubSubHandler (code that manages disks and VMs)
jgainerdewar Jul 7, 2025
68a3008
Remove config and message types for AzurePubSubHandler
jgainerdewar Jul 7, 2025
75c7b9b
Remove WsmDao (legacy WSM interface)
jgainerdewar Jul 7, 2025
52e602f
Remove WsmClientProvider (modern WSM interface)
jgainerdewar Jul 7, 2025
3e093f9
Remove BpmClientProvider
jgainerdewar Jul 7, 2025
ae6c629
Remove WSM and BPM config
jgainerdewar Jul 7, 2025
5dd85e2
Remove ListenerDao
jgainerdewar Jul 7, 2025
2dac410
Remove remaining Listener references and config
jgainerdewar Jul 7, 2025
0ee47ec
Delete more Azure models
jgainerdewar Jul 7, 2025
c176698
Remove Azure VM scripts
jgainerdewar Jul 7, 2025
f600fd8
Clean up LeoMetricsMonitor and related code (tests TBD)
jgainerdewar Jul 7, 2025
aed9cbe
Clean up LeoMetricsMonitorSpec
jgainerdewar Jul 7, 2025
af0313d
scalafmt
jgainerdewar Jul 8, 2025
a9b389a
Get tests passing
jgainerdewar Jul 8, 2025
e37ea45
Remove unused services from baseline dependencies
jgainerdewar Jul 8, 2025
fb00ad0
Remove unused route models
jgainerdewar Jul 8, 2025
81a4cc4
Remove WSM and BPM dependencies
jgainerdewar Jul 8, 2025
eea428e
Remove most v2 endpoints from swagger doc
jgainerdewar Jul 8, 2025
4f27c44
Test support cleanup
jgainerdewar Jul 8, 2025
fd9fca1
TODO app update
jgainerdewar Jul 9, 2025
c823dbf
Scalafmt
jgainerdewar Jul 9, 2025
03cefaa
Try adding removed dependencies back in
jgainerdewar Jul 10, 2025
801a207
Actually add removed dependencies back in
jgainerdewar Jul 10, 2025
1f47e52
Import cleanup
jgainerdewar Jul 11, 2025
95de5b7
Restore cloud-agnostic v2 list runtimes route
jgainerdewar Jul 11, 2025
8a201e3
Scalafmt
jgainerdewar Jul 11, 2025
7dd092d
Remove Azure swatomation tests
jgainerdewar Jul 15, 2025
7859d28
Retry removing WSM, BPM clients
jgainerdewar Jul 16, 2025
0982036
Don't exclude opentelemetry from TCL
jgainerdewar Jul 16, 2025
7707c9a
Restore v2 start and stop endpoints
jgainerdewar Jul 16, 2025
595e0de
Fix route structure
jgainerdewar Jul 17, 2025
e11000d
Merge branch 'develop' into jd_AN-649_removeAzureCode
jgainerdewar Jul 21, 2025
94fc0ff
PR feedback
jgainerdewar Jul 22, 2025
9861291
Merge branch 'jd_AN-649_removeAzureCode' of github.com:DataBiosphere/…
jgainerdewar Jul 22, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package org.broadinstitute.dsde.workbench.leonardo

import cats.effect.{IO, Resource}
import com.azure.resourcemanager.compute.models.VirtualMachineSizeTypes
import org.broadinstitute.dsde.workbench.DoneCheckable
import org.broadinstitute.dsde.workbench.DoneCheckableSyntax._
import org.broadinstitute.dsde.workbench.google2.{
Expand Down Expand Up @@ -135,19 +134,6 @@ object LeonardoApiClient {
None
)

val defaultCreateAzureRuntimeRequest = CreateAzureRuntimeRequest(
Map.empty,
VirtualMachineSizeTypes.STANDARD_DS1_V2,
Map.empty,
CreateAzureDiskRequest(
Map.empty,
AzureDiskName(UUID.randomUUID().toString.substring(0, 8)),
None,
None
),
Some(0)
)

def createRuntime(
googleProject: GoogleProject,
runtimeName: RuntimeName,
Expand Down Expand Up @@ -685,101 +671,6 @@ object LeonardoApiClient {
}
} yield r

def createAzureRuntime(
workspaceId: WorkspaceId,
runtimeName: RuntimeName,
useExistingDisk: Boolean,
createAzureRuntimeRequest: CreateAzureRuntimeRequest = defaultCreateAzureRuntimeRequest
)(implicit client: Client[IO], authorization: IO[Authorization]): IO[Unit] =
for {
traceIdHeader <- genTraceIdHeader()
authHeader <- authorization
r <- client
.run(
Request[IO](
method = Method.POST,
headers = Headers(authHeader, defaultMediaType, traceIdHeader),
uri = rootUri
.withPath(
Uri.Path
.unsafeFromString(s"/api/v2/runtimes/${workspaceId.value.toString}/azure/${runtimeName.asString}")
)
.withQueryParam("useExistingDisk", useExistingDisk),
entity = createAzureRuntimeRequest
)
)
.use { resp =>
if (!resp.status.isSuccess) {
onError(s"Failed to create runtime ${workspaceId.value.toString}/${runtimeName.asString}")(resp)
.flatMap(IO.raiseError)
} else
IO.unit
}
} yield ()

def getAzureRuntime(
workspaceId: WorkspaceId,
runtimeName: RuntimeName
)(implicit client: Client[IO], authorization: IO[Authorization]): IO[GetRuntimeResponseCopy] =
for {
traceIdHeader <- genTraceIdHeader()
authHeader <- authorization
r <- client.expectOr[GetRuntimeResponseCopy](
Request[IO](
method = Method.GET,
headers = Headers(authHeader, traceIdHeader),
uri = rootUri.withPath(
Uri.Path.unsafeFromString(s"/api/v2/runtimes/${workspaceId.value.toString}/azure/${runtimeName.asString}")
)
)
)(onError(s"Failed to get runtime ${workspaceId.value.toString}/${runtimeName.asString}"))
} yield r

def deleteRuntimeV2(
workspaceId: WorkspaceId,
runtimeName: RuntimeName,
deleteDisk: Boolean = true
)(implicit client: Client[IO], authorization: IO[Authorization]): IO[Unit] =
for {
traceIdHeader <- genTraceIdHeader()
authHeader <- authorization
r <- client
.run(
Request[IO](
method = Method.DELETE,
headers = Headers(authHeader, traceIdHeader),
uri = rootUri
.withPath(
Uri.Path
.unsafeFromString(s"/api/v2/runtimes/${workspaceId.value.toString}/azure/${runtimeName.asString}")
)
.withQueryParam("deleteDisk", deleteDisk)
)
)
.use { resp =>
if (!resp.status.isSuccess) {
onError(s"Failed to delete runtime ${workspaceId.value.toString}/${runtimeName.asString}")(resp)
.flatMap(IO.raiseError)
} else
IO.unit
}
} yield r

// TODO: delete this
def deleteRuntimeV2WithWait(
workspaceId: WorkspaceId,
runtimeName: RuntimeName,
deleteDisk: Boolean = true
)(implicit client: Client[IO], authorization: IO[Authorization]): IO[Unit] =
for {
_ <- deleteRuntimeV2(workspaceId, runtimeName, deleteDisk)
ioa = getAzureRuntime(workspaceId, runtimeName).attempt
res <- IO.sleep(20 seconds) >> streamFUntilDone(ioa, 50, 5 seconds).compile.lastOrError
_ <-
if (res.isDone) IO.unit
else IO.raiseError(new TimeoutException(s"delete runtime ${workspaceId}/${runtimeName.asString}"))
} yield ()

def testSparkWebUi(
googleProject: GoogleProject,
runtimeName: RuntimeName,
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,6 @@ object LeonardoConfig extends CommonConfig {
val leonardoServiceAccountUsername = gcs.getString("leonardoServiceAccountUsername")
}

object Azure {
val vmUser = azure.getString("leoVmUser")
val vmPassword = azure.getString("leoVmPassword")
val bastionName = azure.getString("bastionName")
val defaultBastionPort = azure.getInt("defaultBastionPort")
}

// TODO: this should be updated once we're able to run azure automation tests as part of CI
object WSM {
val wsmUri: String = "https://workspace.dsde-dev.broadinstitute.org"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,13 @@ package org.broadinstitute.dsde.workbench.leonardo
import cats.effect.{IO, Resource}
import com.google.cloud.oslogin.common.OsLoginProto.SshPublicKey
import com.google.cloud.oslogin.v1.{ImportSshPublicKeyRequest, OsLoginServiceClient}
import org.broadinstitute.dsde.rawls.model.AzureManagedAppCoordinates
import org.typelevel.log4cats.StructuredLogger
import org.typelevel.log4cats.slf4j.Slf4jLogger
import net.schmizz.sshj.SSHClient
import net.schmizz.sshj.connection.channel.direct.Session
import net.schmizz.sshj.transport.verification.PromiscuousVerifier
import org.broadinstitute.dsde.workbench.model.WorkbenchEmail
import org.broadinstitute.dsde.workbench.model.google.GoogleProject
import org.typelevel.log4cats.StructuredLogger
import org.typelevel.log4cats.slf4j.Slf4jLogger

import java.nio.file.{Files, Path, Paths}
import java.util.UUID
Expand All @@ -27,33 +26,6 @@ case class Tunnel(pid: String, port: Int) {
object SSH {
val loggerIO: StructuredLogger[IO] = Slf4jLogger.getLogger[IO]

// TODO: If multiple tests need to ssh into an azure VM: add a lock of sorts, only one tunnel at a time with same port
// A bastion tunnel is needed to tunnel to an azure vm
// See: https://learn.microsoft.com/en-us/azure/bastion/native-client
def startAzureBastionTunnel(runtimeName: RuntimeName, port: Int = LeonardoConfig.Azure.defaultBastionPort)(implicit
staticTestCoordinates: AzureManagedAppCoordinates
): Resource[IO, Tunnel] = {
val targetResourceId =
s"/subscriptions/${staticTestCoordinates.subscriptionId.toString}/resourceGroups/${staticTestCoordinates.managedResourceGroupId}/providers/Microsoft.Compute/virtualMachines/${runtimeName.asString}"

val makeTunnel = for {
scriptPath <- IO(getClass.getClassLoader.getResource("startTunnel.sh").getPath)
process = Process(
scriptPath,
None,
"BASTION_NAME" -> LeonardoConfig.Azure.bastionName,
"RESOURCE_GROUP" -> staticTestCoordinates.managedResourceGroupId,
"RESOURCE_ID" -> targetResourceId,
"PORT" -> port.toString
)
output <- IO(process !!)
_ <- loggerIO.info(s"Bastion tunnel start command full output:\n\t${output}")
tunnel = Tunnel(output.split('\n').last, port)
} yield tunnel

Resource.make(makeTunnel)(tunnel => loggerIO.info("Closing tunnel") >> closeTunnel(tunnel))
}

final case class SSHSession(session: Session, client: SSHClient)
// Note that a session is a one time use resource, and only supports one command execution
// This method starts an ssh session to either an azure or google runtime.
Expand All @@ -72,18 +44,12 @@ object SSH {
_ <- IO(client.connect(hostName, port))

_ <- loggerIO.info("Authenticating ssh client ")
_ <-
if (sshConfig.cloudProvider == CloudProvider.Azure)
IO(client.authPassword(LeonardoConfig.Azure.vmUser, LeonardoConfig.Azure.vmPassword))
else {
for {
keyConfig <- createSSHKeys(WorkbenchEmail(LeonardoConfig.Leonardo.serviceAccountEmail),
sshConfig.googleProject.get
)
_ <- IO(client.authPublickey(keyConfig.username, keyConfig.privateKey.toAbsolutePath.toString))
} yield ()
}

_ <- for {
keyConfig <- createSSHKeys(WorkbenchEmail(LeonardoConfig.Leonardo.serviceAccountEmail),
sshConfig.googleProject.get
)
_ <- IO(client.authPublickey(keyConfig.username, keyConfig.privateKey.toAbsolutePath.toString))
} yield ()
_ <- loggerIO.info("Starting ssh session")
session <- IO(client.startSession())
} yield SSHSession(session, client)
Expand Down
Loading
Loading