Skip to content

75: get periods run numbers instead of dataset runnumbers during upload #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mytkom merged 2 commits into from
Dec 31, 2024
Merged

75: get periods run numbers instead of dataset runnumbers during upload #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3084094
clean up service tests init functions
mytkom Dec 30, 2024
cf57e43
get timestamp for boundary runs of datasets' periods
mytkom Dec 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cypress/e2e/training-tasks.cy.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ describe('Training Tasks Management', () => {

const testName = generateUniqueName('tm');
cy.get('input[name="name"]').type(testName);
cy.get('select[name="trainingDatasetId"]').select('LHC24b1b');
cy.get('select[name="trainingDatasetId"]').select('1');
cy.get('button').click();

let tmObject = cy.contains('tr', testName)
Expand Down Expand Up @@ -60,7 +60,7 @@ describe('Training Tasks Management', () => {
cy.get('@alreadyExisting').then(alreadyExisting => {
cy.get('input[name="name"]').type(alreadyExisting);
})
cy.get('select[name="trainingDatasetId"]').select('LHC24b1b');
cy.get('select[name="trainingDatasetId"]').select('1');
cy.get('button').click();

cy.get('#errors').invoke('text').should('eq', 'Name must be unique\n')
Expand Down
89 changes: 79 additions & 10 deletions internal/db/migrate/migrate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,87 @@ func SeedDB(db *gorm.DB) error {
return err
}

aods := []jalien.AODFile{{
Name: "AO2D.root",
Path: "/alice/sim/2024/LHC24b1b/0/567454/AOD/002/AO2D.root",
Size: 2312421213,
LHCPeriod: "LHC24b1b",
RunNumber: 567454,
AODNumber: 2,
}}
c1Aods := []jalien.AODFile{
{
Name: "AO2D.root",
Path: "/alice/sim/2023/LHC23c1/302004/AOD/010/AO2D.root",
Size: 3266476446,
LHCPeriod: "LHC23c1",
RunNumber: 302004,
AODNumber: 10,
},
{
Name: "AO2D.root",
Path: "/alice/sim/2023/LHC23c1/302004/AOD/011/AO2D.root",
Size: 3239114872,
LHCPeriod: "LHC23c1",
RunNumber: 302004,
AODNumber: 11,
},
{
Name: "AO2D.root",
Path: "/alice/sim/2023/LHC23c1/302004/AOD/013/AO2D.root",
Size: 3260265579,
LHCPeriod: "LHC23c1",
RunNumber: 302004,
AODNumber: 13,
},
}

mixedPeriodsAods := []jalien.AODFile{
{
Name: "AO2D.root",
Path: "/alice/sim/2023/LHC23c1/302004/AOD/010/AO2D.root",
Size: 3266476446,
LHCPeriod: "LHC23c1",
RunNumber: 302004,
AODNumber: 10,
},
{
Name: "AO2D.root",
Path: "/alice/sim/2023/LHC23c1/302004/AOD/011/AO2D.root",
Size: 3239114872,
LHCPeriod: "LHC23c1",
RunNumber: 302004,
AODNumber: 11,
},
{
Name: "AO2D.root",
Path: "/alice/sim/2023/LHC23c1/302004/AOD/013/AO2D.root",
Size: 3260265579,
LHCPeriod: "LHC23c1",
RunNumber: 302004,
AODNumber: 13,
},
{
Name: "AO2D.root",
Path: "/alice/sim/2023/LHC23e1/302002/AOD/013/AO2D.root",
Size: 35403114,
LHCPeriod: "LHC23e1",
RunNumber: 302002,
AODNumber: 13,
},
{
Name: "AO2D.root",
Path: "/alice/sim/2023/LHC23e1/302002/AOD/024/AO2D.root",
Size: 97906832,
LHCPeriod: "LHC23e1",
RunNumber: 302002,
AODNumber: 24,
},
{
Name: "AO2D.root",
Path: "/alice/sim/2023/LHC23e1/302002/AOD/030/AO2D.root",
Size: 175726295,
LHCPeriod: "LHC23e1",
RunNumber: 302002,
AODNumber: 30,
},
}

trainingDatasets := []models.TrainingDataset{
{Name: "LHC24b1b", AODFiles: aods, UserId: users[0].ID},
{Name: "LHC24b1b2", AODFiles: aods, UserId: users[1].ID},
{Name: "Mixed periods 2023", AODFiles: mixedPeriodsAods, UserId: users[0].ID},
{Name: "LHC23c1", AODFiles: c1Aods, UserId: users[1].ID},
}

if err := db.Save(trainingDatasets).Error; err != nil {
Expand Down
4 changes: 2 additions & 2 deletions internal/handler/training_task_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,10 +172,10 @@ func (h *TrainingTaskHandler) Create(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusCreated)
}

func InitTrainingTaskRoutes(mux *http.ServeMux, env *environment.Env, ccdbService service.ICCDBService, fileService service.IFileService, nnArch service.INNArchService) {
func InitTrainingTaskRoutes(mux *http.ServeMux, env *environment.Env, ccdbService service.ICCDBService, jalienService service.IJAliEnService, fileService service.IFileService, nnArch service.INNArchService) {
prefix := "training-tasks"

ttService := service.NewTrainingTaskService(env.RepositoryContext, ccdbService, fileService, nnArch)
ttService := service.NewTrainingTaskService(env.RepositoryContext, ccdbService, jalienService, fileService, nnArch)
tjh := NewTrainingTaskHandler(env, ttService)

authMw := middleware.NewAuthMw(env.IAuthService, true)
Expand Down
2 changes: 1 addition & 1 deletion internal/jalien/commands.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ func ListAndParseDirectory(path string) (*DirectoryContents, error) {

if lineParsed.IsDir {
dirContents.Subdirs = append(dirContents.Subdirs, Dir{
Name: lineParsed.Name,
Name: strings.TrimSuffix(lineParsed.Name, "/"),
Path: linePath,
})
} else if lineParsed.Name == aodFilename {
Expand Down
4 changes: 2 additions & 2 deletions internal/jalien/commands_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func TestParseLongFormat(t *testing.T) {
},
{
name: "Valid directory line",
input: "drwxr-xr-x user group 4096 Feb 15 08:00 somedir",
input: "drwxr-xr-x user group 4096 Feb 15 08:00 somedir/",
want: &longFormatParsed{
Permissions: "drwxr-xr-x",
Owner: "user",
Expand All @@ -38,7 +38,7 @@ func TestParseLongFormat(t *testing.T) {
Month: "Feb",
Day: "15",
Time: "08:00",
Name: "somedir",
Name: "somedir/",
IsDir: true,
},
wantErr: false,
Expand Down
2 changes: 1 addition & 1 deletion internal/router.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func NewRouter(cfg *config.Config, repoContext *repository.RepositoryContext, au
// handlers' routes
handler.InitLandingRoutes(mux, env)
handler.InitTrainingDatasetRoutes(mux, env, jalienService)
handler.InitTrainingTaskRoutes(mux, env, ccdbService, fileService, nnArch)
handler.InitTrainingTaskRoutes(mux, env, ccdbService, jalienService, fileService, nnArch)
handler.InitTrainingMachineRoutes(mux, env, hasher)
handler.InitQueueRoutes(mux, env, fileService, hasher)

Expand Down
114 changes: 99 additions & 15 deletions internal/service/training_task_service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,14 @@ import (
"errors"
"fmt"
"log"
"regexp"
"slices"
"strconv"

"github.com/mytkom/AliceTraINT/internal/ccdb"
"github.com/mytkom/AliceTraINT/internal/db/models"
"github.com/mytkom/AliceTraINT/internal/db/repository"
"github.com/mytkom/AliceTraINT/internal/jalien"
"gorm.io/gorm"
)

Expand All @@ -32,17 +36,21 @@ type ITrainingTaskService interface {

type TrainingTaskService struct {
*repository.RepositoryContext
CCDBService ICCDBService
FileService IFileService
NNArch INNArchService
CCDBService ICCDBService
JAliEnService IJAliEnService
FileService IFileService
NNArch INNArchService
PeriodRegex *regexp.Regexp
}

func NewTrainingTaskService(repo *repository.RepositoryContext, ccdbService ICCDBService, fileService IFileService, nnArch INNArchService) *TrainingTaskService {
func NewTrainingTaskService(repo *repository.RepositoryContext, ccdbService ICCDBService, jalienService IJAliEnService, fileService IFileService, nnArch INNArchService) *TrainingTaskService {
return &TrainingTaskService{
RepositoryContext: repo,
CCDBService: ccdbService,
JAliEnService: jalienService,
FileService: fileService,
NNArch: nnArch,
PeriodRegex: regexp.MustCompile(`(/alice/sim/\d{4}/LHC[a-z0-9A-Z\_].+(/\d+)?)/\d+/AOD/\d+`),
}
}

Expand Down Expand Up @@ -152,14 +160,41 @@ func (s *TrainingTaskService) UploadOnnxResults(id uint) error {
}
}

smallestRun, greatestRun, err := s.findRunNumberRange(trainingTask)
lhcPeriods, err := s.getLHCPeriods(trainingTask)
if err != nil {
return err
}

firstRunInfo, lastRunInfo, err := s.getRunInfoRange(smallestRun, greatestRun)
if err != nil {
return err
var minSOR, maxEOR uint64
initialized := false

for i, period := range lhcPeriods {
log.Printf("%d: Name=\"%s\" DirPath=\"%s\"", i, period.Name, period.DirPath)

dirContents, err := s.JAliEnService.ListAndParseDirectory(period.DirPath)
if err != nil {
return err
}

smallestRun, greatestRun, err := s.findRunNumberRange(dirContents.Subdirs)
if err != nil {
return err
}

firstRunInfo, lastRunInfo, err := s.getRunInfoRange(smallestRun, greatestRun)
if err != nil {
return err
}

if !initialized || firstRunInfo.SOR < minSOR {
minSOR = firstRunInfo.SOR
}

if !initialized || lastRunInfo.EOR > maxEOR {
maxEOR = lastRunInfo.EOR
}

initialized = true
}

mappedOnnxFiles, err := s.filterOnnxFiles(trainingTask.ID)
Expand All @@ -168,7 +203,7 @@ func (s *TrainingTaskService) UploadOnnxResults(id uint) error {
}

for uploadName, file := range mappedOnnxFiles {
if err := s.uploadOnnxFile(firstRunInfo.SOR, lastRunInfo.EOR, file, uploadName); err != nil {
if err := s.uploadOnnxFile(minSOR, maxEOR, file, uploadName); err != nil {
return err
}
}
Expand All @@ -181,16 +216,65 @@ func (s *TrainingTaskService) UploadOnnxResults(id uint) error {
return nil
}

func (s *TrainingTaskService) findRunNumberRange(task *models.TrainingTask) (uint64, uint64, error) {
var smallestRun, greatestRun uint64
type lhcPeriod struct {
Name string
DirPath string
}

func (s *TrainingTaskService) periodPathFromAODPath(aodPath string) (string, error) {
matches := s.PeriodRegex.FindStringSubmatch(aodPath)

if len(matches) != 3 {
return "", errors.New("unexpected AOD path format, cannot correctly match")
}

return matches[1], nil
}

func (s *TrainingTaskService) getLHCPeriods(task *models.TrainingTask) ([]lhcPeriod, error) {
var periods []lhcPeriod
initialized := false

for _, aod := range task.TrainingDataset.AODFiles {
if !initialized || aod.RunNumber < smallestRun {
smallestRun = aod.RunNumber
if !slices.ContainsFunc(periods, func(p lhcPeriod) bool {
return p.Name == aod.LHCPeriod
}) {
periodPath, err := s.periodPathFromAODPath(aod.Path)
if err != nil {
return nil, err
}

periods = append(periods, lhcPeriod{
Name: aod.LHCPeriod,
DirPath: periodPath,
})
}
initialized = true
}

if !initialized {
return nil, errors.New("unexpected behaviour: empty training dataset")
}

return periods, nil
}

func (s *TrainingTaskService) findRunNumberRange(subdirs []jalien.Dir) (uint64, uint64, error) {
var smallestRun, greatestRun uint64
initialized := false

for _, dir := range subdirs {
runNumber, err := strconv.ParseUint(dir.Name, 10, 64)
if err != nil {
log.Println(err.Error())
continue
}

if !initialized || runNumber < smallestRun {
smallestRun = runNumber
}
if !initialized || aod.RunNumber > greatestRun {
greatestRun = aod.RunNumber
if !initialized || runNumber > greatestRun {
greatestRun = runNumber
}
initialized = true
}
Expand Down
2 changes: 1 addition & 1 deletion test/integration/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func mockRouter(db *gorm.DB, cfg *config.Config) *IntegrationTestUtils {
// handlers' routes
handler.InitLandingRoutes(mux, env)
handler.InitTrainingDatasetRoutes(mux, env, jalienService)
handler.InitTrainingTaskRoutes(mux, env, ccdbService, fileService, nnArch)
handler.InitTrainingTaskRoutes(mux, env, ccdbService, jalienService, fileService, nnArch)
handler.InitTrainingMachineRoutes(mux, env, hasher)
handler.InitQueueRoutes(mux, env, fileService, hasher)

Expand Down
16 changes: 12 additions & 4 deletions test/integration/training_task_handler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,13 +293,21 @@ func prepareUploadToCCDB(t *testing.T, ut *IntegrationTestUtils, user *models.Us
assert.NoError(t, ut.TrainingTask.Create(trainingTask))

now := uint64(time.Now().UTC().UnixMilli())
ut.MockedServices.CCDB.On("GetRunInformation", td.AODFiles[0].RunNumber).Return(&ccdb.RunInformation{
RunNumber: td.AODFiles[0].RunNumber,
ut.MockedServices.JAliEn.On("ListAndParseDirectory", "/alice/sim/2024/LHC24b1b/0").Return(&jalien.DirectoryContents{
Subdirs: []jalien.Dir{
{Name: "560000", Path: "/alice/sim/2024/LHC24b1b/0/560000"},
{Name: "567454", Path: "/alice/sim/2024/LHC24b1b/0/567454"},
{Name: "567458", Path: "/alice/sim/2024/LHC24b1b/0/567458"},
{Name: "570000", Path: "/alice/sim/2024/LHC24b1b/0/570000"},
},
}, nil)
ut.MockedServices.CCDB.On("GetRunInformation", uint64(560000)).Return(&ccdb.RunInformation{
RunNumber: 560000,
SOR: now - 10000,
EOR: now,
}, nil)
ut.MockedServices.CCDB.On("GetRunInformation", td.AODFiles[1].RunNumber).Return(&ccdb.RunInformation{
RunNumber: td.AODFiles[1].RunNumber,
ut.MockedServices.CCDB.On("GetRunInformation", uint64(570000)).Return(&ccdb.RunInformation{
RunNumber: 570000,
SOR: now,
EOR: now + 10000,
}, nil)
Expand Down
Loading
Loading