Skip to content

azure container and training module compatibility adjustments #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mytkom merged 2 commits into from
Jan 5, 2025
Merged

azure container and training module compatibility adjustments #22

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0dec8d4
azure container and training module compatibility adjustments
mytkom Jan 5, 2025
61fa749
adjust training task tests
mytkom Jan 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@ bin
gridCertificate.p12
data
**/__debug*
.vscode
.vscode
docker-for-azure/activate
8 changes: 5 additions & 3 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ COPY go.mod go.sum ./
RUN go mod download

# install tailwind CLI
ENV TAILWIND_VERSION=v3.3.2
ENV TAILWIND_VERSION=v3.4.10
RUN curl -L "https://github.com/tailwindlabs/tailwindcss/releases/download/$TAILWIND_VERSION/tailwindcss-linux-x64" -o /usr/local/bin/tailwindcss
RUN chmod +x /usr/local/bin/tailwindcss

Expand Down Expand Up @@ -47,13 +47,15 @@ RUN mkdir ~/.globus
RUN openssl pkcs12 -clcerts -nokeys -in ./gridCertificate.p12 -out ~/.globus/usercert.pem -password pass:
RUN openssl pkcs12 -nocerts -nodes -in ./gridCertificate.p12 -out ~/.globus/userkey.pem -password pass:
RUN chmod 0400 ~/.globus/userkey.pem
ENV CCDB_SSL_CERT_PATH=/root/.globus/usercert.pem
ENV CCDB_SSL_KEY_PATH=/root/.globus/userkey.pem

WORKDIR /app
RUN alien.py getCAcerts
# for some reason first execution of ls is returning nil - from second and then on it works great
# for some reason first execution of ls is returning nil - from second and on it works as expected
RUN alien_ls /

COPY --from=builder /app/.env /app/AliceTraINT .
COPY --from=builder /app/.env /app/AliceTraINT ./
COPY --from=builder /app/web ./web
COPY --from=builder /app/static/ ./static/

Expand Down
5 changes: 5 additions & 0 deletions docker-for-azure/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# run build.sh from docker-for-azure container
FROM mytkom/alicetraint

WORKDIR /app
COPY root.crt .
6 changes: 6 additions & 0 deletions docker-for-azure/build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/usr/bin/env bash

# Run only from docker-for-azure subdir
docker build -t mytkom/alicetraint ../
docker build -t mytkom/alicetraint-for-azure .
docker tag mytkom/alicetraint-for-azure $IMAGE_NAME
51 changes: 51 additions & 0 deletions docker-for-azure/deploy_container.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/usr/bin/env bash

# ENVs to fill
# RESOURCE_GROUP=
# STORAGE_ACCOUNT_NAME=
# LOCATION=
# SHARE_NAME=
# ALICETRAINT_ACR_NAME=
# ALICETRAINT_ACR_URL=
# IMAGE_NAME=
# STORAGE_KEY=$(az storage account keys list --resource-group $RESOURCE_GROUP --account-name $STORAGE_ACCOUNT_NAME --query "[0].value" --output tsv)
# SERVICE_PRINCIPAL_USERNAME=
# SERVICE_PRINCIPAL_PASSWORD=
# PGHOST=
# PGUSER=
# PGPORT=
# PGDATABASE=
# PGPASSWORD=
# CERN_REDIRECT_URL=
# SSL_ROOT_CERT_PATH=

# Before this script
# 1. Log In to acr
# 2. Set all envs
# 3. Run build.sh

# Deploy container
az container create --resource-group $RESOURCE_GROUP \
--name alicetraint \
--image $IMAGE_NAME \
--dns-name-label alicetraint \
--ports 80 \
--cpu 1 \
--memory 1 \
--registry-username $SERVICE_PRINCIPAL_USERNAME \
--registry-password $SERVICE_PRINCIPAL_PASSWORD \
--azure-file-volume-account-name $STORAGE_ACCOUNT_NAME \
--azure-file-volume-account-key $STORAGE_KEY \
--azure-file-volume-share-name $SHARE_NAME \
--azure-file-volume-mount-path /app/data/ \
--secure-environment-variables \
'DB_PASSWORD'=$PGPASSWORD \
--environment-variables \
'DB_HOST'=$PGHOST \
'DB_PORT'=$PGPORT \
'DB_USER'=$PGUSER \
'DB_NAME'=$PGDATABASE \
'CERN_REDIRECT_URL'=$CERN_REDIRECT_URL \
'ALICETRAINT_PORT'='80' \
'DB_SSL_CERT_PATH'=$SSL_ROOT_CERT_PATH \
'DB_SSLMODE'='verify-full'
33 changes: 33 additions & 0 deletions docker-for-azure/root.crt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
-----BEGIN CERTIFICATE-----
MIIFqDCCA5CgAwIBAgIQHtOXCV/YtLNHcB6qvn9FszANBgkqhkiG9w0BAQwFADBl
MQswCQYDVQQGEwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYw
NAYDVQQDEy1NaWNyb3NvZnQgUlNBIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5
IDIwMTcwHhcNMTkxMjE4MjI1MTIyWhcNNDIwNzE4MjMwMDIzWjBlMQswCQYDVQQG
EwJVUzEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTYwNAYDVQQDEy1N
aWNyb3NvZnQgUlNBIFJvb3QgQ2VydGlmaWNhdGUgQXV0aG9yaXR5IDIwMTcwggIi
MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDKW76UM4wplZEWCpW9R2LBifOZ
Nt9GkMml7Xhqb0eRaPgnZ1AzHaGm++DlQ6OEAlcBXZxIQIJTELy/xztokLaCLeX0
ZdDMbRnMlfl7rEqUrQ7eS0MdhweSE5CAg2Q1OQT85elss7YfUJQ4ZVBcF0a5toW1
HLUX6NZFndiyJrDKxHBKrmCk3bPZ7Pw71VdyvD/IybLeS2v4I2wDwAW9lcfNcztm
gGTjGqwu+UcF8ga2m3P1eDNbx6H7JyqhtJqRjJHTOoI+dkC0zVJhUXAoP8XFWvLJ
jEm7FFtNyP9nTUwSlq31/niol4fX/V4ggNyhSyL71Imtus5Hl0dVe49FyGcohJUc
aDDv70ngNXtk55iwlNpNhTs+VcQor1fznhPbRiefHqJeRIOkpcrVE7NLP8TjwuaG
YaRSMLl6IE9vDzhTyzMMEyuP1pq9KsgtsRx9S1HKR9FIJ3Jdh+vVReZIZZ2vUpC6
W6IYZVcSn2i51BVrlMRpIpj0M+Dt+VGOQVDJNE92kKz8OMHY4Xu54+OU4UZpyw4K
UGsTuqwPN1q3ErWQgR5WrlcihtnJ0tHXUeOrO8ZV/R4O03QK0dqq6mm4lyiPSMQH
+FJDOvTKVTUssKZqwJz58oHhEmrARdlns87/I6KJClTUFLkqqNfs+avNJVgyeY+Q
W5g5xAgGwax/Dj0ApQIDAQABo1QwUjAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/
BAUwAwEB/zAdBgNVHQ4EFgQUCctZf4aycI8awznjwNnpv7tNsiMwEAYJKwYBBAGC
NxUBBAMCAQAwDQYJKoZIhvcNAQEMBQADggIBAKyvPl3CEZaJjqPnktaXFbgToqZC
LgLNFgVZJ8og6Lq46BrsTaiXVq5lQ7GPAJtSzVXNUzltYkyLDVt8LkS/gxCP81OC
gMNPOsduET/m4xaRhPtthH80dK2Jp86519efhGSSvpWhrQlTM93uCupKUY5vVau6
tZRGrox/2KJQJWVggEbbMwSubLWYdFQl3JPk+ONVFT24bcMKpBLBaYVu32TxU5nh
SnUgnZUP5NbcA/FZGOhHibJXWpS2qdgXKxdJ5XbLwVaZOjex/2kskZGT4d9Mozd2
TaGf+G0eHdP67Pv0RR0Tbc/3WeUiJ3IrhvNXuzDtJE3cfVa7o7P4NHmJweDyAmH3
pvwPuxwXC65B2Xy9J6P9LjrRk5Sxcx0ki69bIImtt2dmefU6xqaWM/5TkshGsRGR
xpl/j8nWZjEgQRCHLQzWwa80mMpkg/sTV9HB8Dx6jKXB/ZUhoHHBk2dxEuqPiApp
GWSZI1b7rCoucL5mxAyE7+WL85MB+GqQk2dLsmijtWKP6T+MejteD+eMuMZ87zf9
dOLITzNy4ZQ5bb0Sr74MTnB8G2+NszKTc0QWbej09+CVgI+WXTik9KveCjCHk9hN
AHFiRSdLOkKEW39lt2c0Ui2cFmuqqNh7o0JMcccMyj6D5KbvtwEwXlGjefVwaaZB
RA+GsCyRxj3qrg+E
-----END CERTIFICATE-----
41 changes: 25 additions & 16 deletions internal/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,13 @@ type Config struct {
}

type DatabaseConfig struct {
Host string
Port uint
User string
Password string
DBName string
SSLMode string
Host string
Port uint
User string
Password string
DBName string
SSLMode string
SSLRootCertPath string
}

func LoadConfig() *Config {
Expand All @@ -38,12 +39,13 @@ func LoadConfig() *Config {

return &Config{
Database: DatabaseConfig{
Host: getEnv("DB_HOST", "localhost"),
Port: getEnvAsUint("DB_PORT", 5432),
User: getEnv("DB_USER", "user"),
Password: getEnv("DB_PASSWORD", "password"),
DBName: getEnv("DB_NAME", "AliceTraINT_db"),
SSLMode: getEnv("DB_SSLMODE", "disable"),
Host: getEnv("DB_HOST", "localhost"),
Port: getEnvAsUint("DB_PORT", 5432),
User: getEnv("DB_USER", "user"),
Password: getEnv("DB_PASSWORD", "password"),
DBName: getEnv("DB_NAME", "AliceTraINT_db"),
SSLMode: getEnv("DB_SSLMODE", "disable"),
SSLRootCertPath: getEnv("DB_SSL_CERT_PATH", ""),
},
Port: getEnv("ALICETRAINT_PORT", "8088"),
JalienCacheMinutes: getEnvAsUint("ALICETRAINT_JALIEN_CACHE_MINUTES", 60),
Expand All @@ -57,10 +59,17 @@ func LoadConfig() *Config {
}

func (dbConfig *DatabaseConfig) ConnectionString() string {
return fmt.Sprintf(
"host=%s port=%d user=%s password=%s dbname=%s sslmode=%s",
dbConfig.Host, dbConfig.Port, dbConfig.User, dbConfig.Password, dbConfig.DBName, dbConfig.SSLMode,
)
if dbConfig.SSLMode == "disabled" {
return fmt.Sprintf(
"host=%s port=%d user=%s password=%s dbname=%s sslmode=%s",
dbConfig.Host, dbConfig.Port, dbConfig.User, dbConfig.Password, dbConfig.DBName, dbConfig.SSLMode,
)
} else {
return fmt.Sprintf(
"host=%s port=%d user=%s password=%s dbname=%s sslmode=%s sslrootcert=%s",
dbConfig.Host, dbConfig.Port, dbConfig.User, dbConfig.Password, dbConfig.DBName, dbConfig.SSLMode, dbConfig.SSLRootCertPath,
)
}
}

func getEnv(key, defaultValue string) string {
Expand Down
21 changes: 13 additions & 8 deletions internal/db/migrate/migrate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"time"

"github.com/mytkom/AliceTraINT/internal/db/models"
"github.com/mytkom/AliceTraINT/internal/hash"
"github.com/mytkom/AliceTraINT/internal/jalien"
"gorm.io/gorm"
)
Expand Down Expand Up @@ -87,25 +88,25 @@ func SeedDB(db *gorm.DB) error {
},
{
Name: "AO2D.root",
Path: "/alice/sim/2023/LHC23e1/302002/AOD/013/AO2D.root",
Path: "/alice/sim/2023/LHC23d4/302005/AOD/013/AO2D.root",
Size: 35403114,
LHCPeriod: "LHC23e1",
LHCPeriod: "LHC23d4",
RunNumber: 302002,
AODNumber: 13,
},
{
Name: "AO2D.root",
Path: "/alice/sim/2023/LHC23e1/302002/AOD/024/AO2D.root",
Path: "/alice/sim/2023/LHC23d4/302005/AOD/024/AO2D.root",
Size: 97906832,
LHCPeriod: "LHC23e1",
LHCPeriod: "LHC23d4",
RunNumber: 302002,
AODNumber: 24,
},
{
Name: "AO2D.root",
Path: "/alice/sim/2023/LHC23e1/302002/AOD/030/AO2D.root",
Path: "/alice/sim/2023/LHC23d4/302005/AOD/030/AO2D.root",
Size: 175726295,
LHCPeriod: "LHC23e1",
LHCPeriod: "LHC23d4",
RunNumber: 302002,
AODNumber: 30,
},
Expand All @@ -120,9 +121,13 @@ func SeedDB(db *gorm.DB) error {
return err
}

secretHashed, err := hash.HashKey("secretkey_secretkey_secretkey_sk")
if err != nil {
return err
}
trainingMachines := []models.TrainingMachine{
{Name: "tm1", LastActivityAt: time.Now(), SecretKeyHashed: "salt:secret", UserId: users[0].ID},
{Name: "tm2", LastActivityAt: time.Now(), SecretKeyHashed: "salt:secret2", UserId: users[1].ID},
{Name: "tm1", LastActivityAt: time.Now(), SecretKeyHashed: secretHashed, UserId: users[0].ID},
{Name: "tm2", LastActivityAt: time.Now(), SecretKeyHashed: secretHashed, UserId: users[1].ID},
}

if err := db.Save(trainingMachines).Error; err != nil {
Expand Down
3 changes: 1 addition & 2 deletions internal/handler/queue_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,12 +109,11 @@ func (qh *QueueHandler) QueryTask(w http.ResponseWriter, r *http.Request) {
Configuration: tt.Configuration,
}

w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(response); err != nil {
http.Error(w, "cannot encode response", http.StatusInternalServerError)
return
}

w.WriteHeader(http.StatusOK)
}

func (qh *QueueHandler) CreateTrainingTaskResult(w http.ResponseWriter, r *http.Request) {
Expand Down
2 changes: 1 addition & 1 deletion internal/service/training_task_service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ func (s *TrainingTaskService) GetByID(id uint) (*TrainingTaskWithResults, error)
}

var imageFiles []models.TrainingTaskResult
if trainingTask.Status >= models.Completed {
if trainingTask.Status >= models.Training {
imageFiles, err = s.TrainingTaskResult.GetByType(trainingTask.ID, models.Image)
if err != nil {
return nil, errInternalServerError
Expand Down
Loading
Loading