Skip to content

[PERF-338] Install applications on Azure VM for performance testing #115

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
josejuancm wants to merge 8 commits into
base: main
Choose a base branch
from
Draft

[PERF-338] Install applications on Azure VM for performance testing #115

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
88fc045
prerequisites updated
josejuancm Jan 16, 2025
40c2bd3
action to download the installation packages and scripts creation
josejuancm Jan 16, 2025
c1d3a14
fixes on references and versions
josejuancm Jan 17, 2025
fd989ef
feat: added missing secrets
Jan 18, 2025
c68cb1f
swagger and sandbox installation scripts
josejuancm Jan 18, 2025
40a287e
Get package versions based on ODS version to install
simpat-jesus Jan 31, 2025
e57ce1f
Update action version
simpat-jesus Jan 31, 2025
1e2f631
Remove action environments
simpat-jesus Jan 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 99 additions & 0 deletions .github/workflows/get-versions.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
#Requires -Version 5
param(
[Parameter(mandatory=$true)]
[string]
$PackageVersion
)

function Invoke-SemanticSort {
param(
[Parameter(Mandatory=$true)]
[string[]]
$Versions
)

$major = @{label="major";expression={[int]$_.Split(".")[0]}}
$minor = @{label="minor";expression={[int]$_.Split(".")[1]}}
$patch = @{label="patch";expression={[int]$_.Split(".")[2]}}

$Versions `
| Select-Object $major, $minor, $patch `
| Sort-Object major, minor, patch -Descending `
| ForEach-Object { "$($_.major).$($_.minor).$($_.patch)" }
}

function Get-NugetPackageVersion {
[CmdletBinding()]
param(
[Parameter(Mandatory=$true)]
[string]
$PackageName,

[Parameter(Mandatory=$true)]
[string]
$PackageVersion,

# URL for the release package feed
[string]
$ReleaseServiceIndex = "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_packaging/EdFi%40Release/nuget/v3/index.json"
)

$nugetServicesURL = $ReleaseServiceIndex

# The first URL just contains metadata for looking up more useful services
$nugetServices = Invoke-RestMethod $nugetServicesURL

$packageService = $nugetServices.resources `
| Where-Object { $_."@type" -like "PackageBaseAddress*" } `
| Select-Object -Property "@id" -ExpandProperty "@id"

# Pad the package version out to three part semver
switch ($PackageVersion.split(".").length) {
1 { $versionSearch = "$PackageVersion.*.*"}
2 { $versionSearch = "$PackageVersion.*" }
3 { $versionSearch = $PackageVersion }
default: { throw "Invalid version string ``$($PackageVersion)``. Should be one, two, or three components from a Semantic Version.".Trim() }
}
$lowerId = $PackageName.ToLower()

# Lookup available packages
$package = Invoke-RestMethod "$($packageService)$($lowerId)/index.json"

# Sort by SemVer
$versions = Invoke-SemanticSort $package.versions

# Find the first available version that matches the requested version
$version = $versions | Where-Object { $_ -like $versionSearch } | Select-Object -First 1

if ($null -eq $version) {
throw "Version ``$($PackageVersion)`` does not exist yet for ``$($PackageName)``."
}

$version
}
Write-Host $PackageVersion
switch -Exact ($PackageVersion)
{
'7.3' { $StandardVersion = "5.2.0" }
'7.2' { $StandardVersion = "5.1.0" }
'7.1' { $StandardVersion = "5.0.0" }
}

$env:WEBAPI_INSTALLLER_VERSION = "$(Get-NugetPackageVersion -PackageName EdFi.Suite3.Installer.WebApi -PackageVersion $PackageVersion)".Trim()
$env:SWAGGER_INSTALLER_VERSION = "$(Get-NugetPackageVersion -PackageName EdFi.Suite3.Installer.SwaggerUI -PackageVersion $PackageVersion)".Trim()
$env:SANDBOXADMIN_INSTALLER_VERSION = "$(Get-NugetPackageVersion -PackageName EdFi.Suite3.Installer.SandboxAdmin -PackageVersion $PackageVersion)".Trim()

$env:SWAGGER_VERSION = "$(Get-NugetPackageVersion -PackageName EdFi.Suite3.Ods.SwaggerUI -PackageVersion $PackageVersion)".Trim()
$env:SANDBOXADMIN_VERSION = "$(Get-NugetPackageVersion -PackageName EdFi.Suite3.Ods.SandboxAdmin -PackageVersion $PackageVersion)".Trim()

if ($null -eq $StandardVersion) {
$env:DATABASE_INSTALLER_VERSION = "$(Get-NugetPackageVersion -PackageName EdFi.Suite3.RestApi.Databases -PackageVersion $PackageVersion)".Trim()
$env:DATABASE_INSTALLER = "EdFi.Suite3.RestApi.Databases"
$env:WEBAPI_VERSION = "$(Get-NugetPackageVersion -PackageName EdFi.Suite3.Ods.WebApi -PackageVersion $PackageVersion)".Trim()
$env:WEBAPI = "EdFi.Suite3.Ods.WebApi"
}else {
$env:DATABASE_INSTALLER_VERSION = "$(Get-NugetPackageVersion -PackageName EdFi.Suite3.RestApi.Databases.Standard.$StandardVersion -PackageVersion $PackageVersion)".Trim()
$env:DATABASE_INSTALLER = "EdFi.Suite3.RestApi.Databases.Standard.$StandardVersion"
$env:WEBAPI_VERSION = "$(Get-NugetPackageVersion -PackageName EdFi.Suite3.Ods.WebApi.Standard.$StandardVersion -PackageVersion $PackageVersion)".Trim()
$env:WEBAPI = "EdFi.Suite3.Ods.WebApi.Standard.$StandardVersion"
}
193 changes: 193 additions & 0 deletions .github/workflows/installation_scripts.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
name: Download packages and create installation scripts

on:
workflow_dispatch:
inputs:
resource_group:
description: "The resource group of the VM"
required: true
web_api_vm_name:
description: "The name of the WEB API VM"
required: true
sql_vm_name:
description: "The name of the SQL VM"
required: true
ods_api_version:
description: "The version of EdFi ODS / API to install"
required: true
default: "7.3"
type: choice
options:
- 7.3
- 7.2
- 7.1
- 6.2
- 5.4
db_server:

@simpat-jesus simpat-jesus Jan 16, 2025

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can the IP be set to a fixed value while provisioning the VM? or use the FDQN instead?

description: "Database server IP address"
required: true

env:
ARM_CLIENT_ID: ${{ vars.AZURE_CLIENT_ID }}
ARM_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
ARM_SUBSCRIPTION_ID: ${{ vars.AZURE_SUBSCRIPTION_ID }}
ARM_TENANT_ID: ${{ vars.AZURE_TENANT_ID }}

jobs:
run-command:
runs-on: ubuntu-latest
steps:

- name: Checkout
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

- name: Get Package Versions
working-directory: ./.github/workflows
run: |
.\get-versions.ps1 -PackageVersion ${{ github.event.inputs.ods_api_version }}
shell: pwsh

- name: Azure Login
uses: azure/login@v1
with:
creds: '{"clientId":"${{ env.ARM_CLIENT_ID }}","clientSecret":"${{ env.ARM_CLIENT_SECRET }}","subscriptionId":"${{ env.ARM_SUBSCRIPTION_ID }}","tenantId":"${{ env.ARM_TENANT_ID }}"}'

- name: Download Components on Web Server VM
run: |
Comment thread
simpat-jesus marked this conversation as resolved.
az vm run-command invoke \
--command-id RunPowerShellScript \
--name "${{ github.event.inputs.web_api_vm_name }}" \
--resource-group "${{ github.event.inputs.resource_group }}" \
--scripts '
# Download and extract WebApi Installer
Invoke-WebRequest -Uri "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_apis/packaging/feeds/EdFi@Release/nuget/packages/EdFi.Suite3.Installer.WebApi/versions/${{ $env:WEBAPI_INSTALLLER_VERSION }}/content" -OutFile "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.WebApi.${{ $env:WEBAPI_INSTALLLER_VERSION }}.nupkg"
Rename-Item -Path "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.WebApi.${{ $env:WEBAPI_INSTALLLER_VERSION }}.nupkg" -NewName "EdFi.Suite3.Installer.WebApi.zip"
Expand-Archive -Path "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.WebApi.zip" -DestinationPath "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.WebApi" -Force

# Download and extract SwaggerUI
Invoke-WebRequest -Uri "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_apis/packaging/feeds/EdFi@Release/nuget/packages/EdFi.Suite3.Installer.SwaggerUI/versions/${{ $env:SWAGGER_INSTALLER_VERSION }}/content" -OutFile "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.SwaggerUI.${{ $env:SWAGGER_INSTALLER_VERSION }}.nupkg"
Rename-Item -Path "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.SwaggerUI.${{ $env:SWAGGER_INSTALLER_VERSION }}.nupkg" -NewName "EdFi.Suite3.Installer.SwaggerUI.zip"
Expand-Archive -Path "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.SwaggerUI.zip" -DestinationPath "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.SwaggerUI" -Force

# Download and extract SandboxAdmin
Invoke-WebRequest -Uri "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_apis/packaging/feeds/EdFi@Release/nuget/packages/EdFi.Suite3.Installer.SandboxAdmin/versions/${{ $env:SANDBOXADMIN_INSTALLER_VERSION }}/content" -OutFile "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.SandboxAdmin.${{ $env:SANDBOXADMIN_INSTALLER_VERSION }}.nupkg"
Rename-Item -Path "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.SandboxAdmin.${{ $env:SANDBOXADMIN_INSTALLER_VERSION }}.nupkg" -NewName "EdFi.Suite3.Installer.SandboxAdmin.zip"
Expand-Archive -Path "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.SandboxAdmin.zip" -DestinationPath "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.SandboxAdmin" -Force'

- name: Download RestApi Databases on SQL Server VM
run: |
az vm run-command invoke \
--command-id RunPowerShellScript \
--name "${{ github.event.inputs.sql_vm_name }}" \
--resource-group "${{ github.event.inputs.resource_group }}" \
--scripts '
# Download and extract RestApi Databases
Invoke-WebRequest -Uri "https://pkgs.dev.azure.com/ed-fi-alliance/Ed-Fi-Alliance-OSS/_apis/packaging/feeds/EdFi@Release/nuget/packages/${{ $env:DATABASE_INSTALLER }}/versions/${{ $env:DATABASE_INSTALLER_VERSION }}/content" -OutFile "C:/ODS/${{ github.event.inputs.ods_api_version }}/${{ $env:DATABASE_INSTALLER }}.nupkg"
Rename-Item -Path "C:/ODS/${{ github.event.inputs.ods_api_version }}/${{ $env:DATABASE_INSTALLER }}.nupkg" -NewName "EdFi.Suite3.RestApi.Databases.Standard.zip"
Expand-Archive -Path "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.RestApi.Databases.Standard.zip" -DestinationPath "C:/ODS/${{ github.event.inputs.ods_api_version }}/${{ $env:DATABASE_INSTALLER }}" -Force
Remove-Item -Path "C:/ODS/${{ github.event.inputs.ods_api_version }}/${{ $env:DATABASE_INSTALLER }}/configuration.postgreSQL.json" -Force'

- name: Create Deployment Script
run: |
SCRIPT_CONTENT='
Set-Location "C:/ODS/${{ github.event.inputs.ods_api_version }}/${{ $env:DATABASE_INSTALLER }}"

Write-Host "Creating ps1 file..."
"Import-Module .\Deployment.psm1" | Out-File -FilePath "dbs_installation.ps1"
"Initialize-DeploymentEnvironment" | Out-File -FilePath "dbs_installation.ps1" -Append

Write-Host "File created successfully"
Get-Content -Path "dbs_installation.ps1"'

az vm run-command invoke \
--command-id RunPowerShellScript \
--name "${{ github.event.inputs.sql_vm_name }}" \
--resource-group "${{ github.event.inputs.resource_group }}" \
--scripts "$SCRIPT_CONTENT"

- name: Create WebApi Installation Script
run: |
SCRIPT_CONTENT='
Set-Location "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.WebApi"

Write-Host "Creating WebApi installation script..."
@"
Import-Module .\Install-EdFiOdsWebApi.psm1
`$parameters = @{
PackageVersion = "${{ $env:WEBAPI_VERSION }}"
PackageName = "${{ $env:WEBAPI }}"
DbConnectionInfo = @{
Engine = "SqlServer"
Server = "${{ github.event.inputs.db_server }}"
Username = "${{ secrets.SQL_ADMIN_USERNAME }}"
Password = "${{ secrets.SQL_ADMIN_PASSWORD }}"
UseIntegratedSecurity = `$false
}
IsSandbox = `$true
UnEncryptedConnection = `$true
}
Install-EdFiOdsWebApi @parameters
"@ | Out-File -FilePath "webapi_installation.ps1"

Write-Host "File created successfully"
Get-Content -Path "webapi_installation.ps1"'

az vm run-command invoke \
--command-id RunPowerShellScript \
--name "${{ github.event.inputs.web_api_vm_name }}" \
--resource-group "${{ github.event.inputs.resource_group }}" \
--scripts "$SCRIPT_CONTENT"

- name: Create SwaggerUI Installation Script
run: |
SCRIPT_CONTENT='
Set-Location "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.SwaggerUI"

Write-Host "Creating ps1 file..."
@"
Import-Module .\Install-EdFiOdsSwaggerUI.psm1
`$parameters = @{
PackageVersion = "${{ $env:SWAGGER_VERSION }}"
WebApiVersionUrl = "https://edfi-ods/WebApi"
PrePopulatedKey = "${{ secrets.POPULATED_KEY }}"
PrePopulatedSecret = "${{ secrets.POPULATED_SECRET }}"
}
Install-EdFiOdsSwaggerUI @parameters
"@ | Out-File -FilePath "swaggerui_installation.ps1"

Write-Host "File created successfully"
Get-Content -Path "swaggerui_installation.ps1"
'

az vm run-command invoke \
--command-id RunPowerShellScript \
--name "${{ github.event.inputs.web_api_vm_name }}" \
--resource-group "${{ github.event.inputs.resource_group }}" \
--scripts "$SCRIPT_CONTENT"

- name: Create SandboxAdmin Installation Script
run: |
SCRIPT_CONTENT='
Set-Location "C:/ODS/${{ github.event.inputs.ods_api_version }}/EdFi.Suite3.Installer.SandboxAdmin"

Write-Host "Creating ps1 file..."
@"
Import-Module .\Install-EdFiOdsSandboxAdmin.psm1
`$parameters = @{
PackageVersion = "${{ $env:SANDBOXADMIN_VERSION }}"
WebApiVersionUrl = "https://edfi-ods/WebApi"
PrePopulatedKey = "${{ secrets.POPULATED_KEY }}"
PrePopulatedSecret = "${{ secrets.POPULATED_SECRET }}"
}
Install-EdFiOdsSandboxAdmin @parameters
"@ | Out-File -FilePath "sandboxadmin_installation.ps1"

Write-Host "File created successfully"
Get-Content -Path "sandboxadmin_installation.ps1"
'

az vm run-command invoke \
--command-id RunPowerShellScript \
--name "${{ github.event.inputs.web_api_vm_name }}" \
--resource-group "${{ github.event.inputs.resource_group }}" \
--scripts "$SCRIPT_CONTENT"
32 changes: 13 additions & 19 deletions .github/workflows/terraform-deploy.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,6 @@ on:
description: 'Environment Label'
required: true
default: 'ods-perf-test'
environment:
description: 'Environment to deploy to'
required: true
type: environment
default: 'development'

permissions:
contents: read
Expand All @@ -34,7 +29,6 @@ jobs:
terraform:
name: 'Terraform'
runs-on: ubuntu-latest
environment: ${{ github.event.inputs.environment }}

defaults:
run:
Expand Down Expand Up @@ -75,7 +69,7 @@ jobs:

- name: Terraform Plan
run: terraform plan -out=tfplan

- name: Terraform Apply
run: terraform apply -auto-approve tfplan

Expand All @@ -85,31 +79,31 @@ jobs:
# Debug: Show terraform state content
echo "Terraform state content:"
cat terraform.tfstate

# Get storage account name using jq for reliable JSON parsing
STORAGE_ACCOUNT=$(cat terraform.tfstate | jq -r '.outputs.azure_storage_tfstate.value')

echo "Extracted storage account name: $STORAGE_ACCOUNT"

# Validate the storage account name
if [ -z "$STORAGE_ACCOUNT" ]; then
echo "Error: Failed to extract storage account name"
exit 1
fi

if [ ${#STORAGE_ACCOUNT} -gt 24 ]; then
echo "Error: Storage account name too long: ${#STORAGE_ACCOUNT} characters"
exit 1
fi

if [[ ! $STORAGE_ACCOUNT =~ ^[a-z0-9]*$ ]]; then
echo "Error: Storage account name contains invalid characters"
exit 1
fi

# Set the output
echo "storage_account=${STORAGE_ACCOUNT}" >> $GITHUB_OUTPUT

# Debug: Show what we're setting
echo "Setting storage account to: ${STORAGE_ACCOUNT}"
shell: bash
Expand All @@ -118,9 +112,9 @@ jobs:
run: |
# Create a more descriptive state file name
STATE_FILE="${{ github.event.inputs.prefix }}-${{ github.event.inputs.label }}.tfstate"

echo "Using state file name: ${STATE_FILE}"

# Create the backend configuration
cat > backend.tf << EOF
terraform {
Expand All @@ -132,7 +126,7 @@ jobs:
}
}
EOF

# Verify the content
echo "Created backend.tf content:"
cat backend.tf
Expand All @@ -141,9 +135,9 @@ jobs:
run: |
echo "Backend configuration:"
cat backend.tf

terraform init -migrate-state -force-copy || {
echo "Migration failed. Current directory contents:"
ls -la
exit 1
}
}
Loading