From a19799ec866d0c66c8ea4e8e43ce5ba339fad672 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Fri, 27 Feb 2026 16:03:37 -0500 Subject: [PATCH 01/19] add tags fields to jobs --- api/handlers/handlers.go | 11 +++- api/jobs/aws_batch_jobs.go | 4 +- api/jobs/database.go | 4 +- api/jobs/database_postgres.go | 57 +++++++++--------- api/jobs/database_sqlite.go | 70 ++++++++++++---------- api/jobs/docker_jobs.go | 10 ++-- api/jobs/jobs.go | 1 + api/jobs/subprocess_jobs.go | 9 +-- api/views/jobs.html | 107 +++++++++++++++++----------------- 9 files changed, 150 insertions(+), 123 deletions(-) diff --git a/api/handlers/handlers.go b/api/handlers/handlers.go index 551cfd7..06fec3e 100644 --- a/api/handlers/handlers.go +++ b/api/handlers/handlers.go @@ -97,6 +97,7 @@ func prepareResponse(c echo.Context, httpStatus int, renderName string, output i // specs: https://developer.ogc.org/api/processes/index.html#tag/Execute type runRequestBody struct { Inputs map[string]interface{} `json:"inputs"` + Tags []string `json:"tags"` } // LandingPage godoc @@ -242,6 +243,7 @@ func (rh *RESTHandler) Execution(c echo.Context) error { StorageSvc: rh.StorageSvc, DB: rh.DB, DoneChan: rh.MessageQueue.JobDone, + Tags: params.Tags, } case "aws-batch": @@ -259,6 +261,7 @@ func (rh *RESTHandler) Execution(c echo.Context) error { StorageSvc: rh.StorageSvc, DB: rh.DB, DoneChan: rh.MessageQueue.JobDone, + Tags: params.Tags, } case "subprocess": @@ -272,6 +275,7 @@ func (rh *RESTHandler) Execution(c echo.Context) error { StorageSvc: rh.StorageSvc, DB: rh.DB, DoneChan: rh.MessageQueue.JobDone, + Tags: params.Tags, } } @@ -574,6 +578,11 @@ func (rh *RESTHandler) ListJobsHandler(c echo.Context) error { processIDs := c.QueryParam("processID") // assuming comma-separated list: "process1,process2" statuses := c.QueryParam("status") submitters := c.QueryParam("submitter") + tagsParam := c.QueryParam("tags") + var tagsList []string + if tagsParam != "" { + tagsList = strings.Split(tagsParam, ",") + } var processIDList []string if processIDs != "" { @@ -617,7 +626,7 @@ func (rh *RESTHandler) ListJobsHandler(c echo.Context) error { offset = 0 } - result, err := rh.DB.GetJobs(limit, offset, processIDList, statusList, submittersList) + result, err := rh.DB.GetJobs(limit, offset, processIDList, statusList, submittersList, tagsList) if err != nil { output := errResponse{HTTPStatus: http.StatusInternalServerError, Message: err.Error()} return prepareResponse(c, http.StatusNotFound, "error", output) diff --git a/api/jobs/aws_batch_jobs.go b/api/jobs/aws_batch_jobs.go index d2b40dc..ac6fc4b 100644 --- a/api/jobs/aws_batch_jobs.go +++ b/api/jobs/aws_batch_jobs.go @@ -38,7 +38,7 @@ type AWSBatchJob struct { UpdateTime time.Time Status string `json:"status"` // results interface{} - + Tags []string `json:"tags"` logger *log.Logger logFile *os.File @@ -251,7 +251,7 @@ func (j *AWSBatchJob) Create() error { j.batchContext = batchContext // At this point job is ready to be added to database - err = j.DB.addJob(j.UUID, "accepted", "", "aws-batch", j.ProcessName, j.Submitter, time.Now()) + err = j.DB.addJob(j.UUID, "accepted", "", "aws-batch", j.ProcessName, j.Submitter, j.Tags, time.Now()) if err != nil { j.ctxCancel() return err diff --git a/api/jobs/database.go b/api/jobs/database.go index ae29a73..3a24e5e 100644 --- a/api/jobs/database.go +++ b/api/jobs/database.go @@ -8,11 +8,11 @@ import ( // Database interface abstracts database operations type Database interface { - addJob(jid, status, mode, host, processID, submitter string, updated time.Time) error + addJob(jid, status, mode, host, processID, submitter string, tags []string, updated time.Time) error updateJobRecord(jid, status string, now time.Time) error GetJob(jid string) (JobRecord, bool, error) CheckJobExist(jid string) (bool, error) - GetJobs(limit, offset int, processIDs, statuses, submitters []string) ([]JobRecord, error) + GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string) ([]JobRecord, error) Close() error } diff --git a/api/jobs/database_postgres.go b/api/jobs/database_postgres.go index 032e2a5..961cc19 100644 --- a/api/jobs/database_postgres.go +++ b/api/jobs/database_postgres.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/lib/pq" _ "github.com/lib/pq" ) @@ -37,21 +38,20 @@ func NewPostgresDB(dbConnString string) (*PostgresDB, error) { func (postgresDB *PostgresDB) createTables() error { queryJobs := ` - CREATE TABLE IF NOT EXISTS jobs ( - id TEXT PRIMARY KEY, - status TEXT NOT NULL, - updated TIMESTAMP WITHOUT TIME ZONE NOT NULL, - mode TEXT NOT NULL, - host TEXT NOT NULL, - process_id TEXT NOT NULL, - submitter TEXT NOT NULL DEFAULT '' - ); - + CREATE TABLE IF NOT EXISTS jobs ( + id TEXT PRIMARY KEY, + status TEXT NOT NULL, + updated TIMESTAMP WITHOUT TIME ZONE NOT NULL, + mode TEXT NOT NULL, + host TEXT NOT NULL, + process_id TEXT NOT NULL, + submitter TEXT NOT NULL DEFAULT '', + tags TEXT[] NOT NULL DEFAULT '{}' + ); CREATE INDEX IF NOT EXISTS idx_jobs_updated ON jobs(updated); CREATE INDEX IF NOT EXISTS idx_jobs_process_id ON jobs(process_id); CREATE INDEX IF NOT EXISTS idx_jobs_submitter ON jobs(submitter); - ` - +` _, err := postgresDB.Handle.Exec(queryJobs) if err != nil { return fmt.Errorf("error creating tables: %s", err) @@ -60,9 +60,9 @@ func (postgresDB *PostgresDB) createTables() error { } // AddJob adds a new job to the database -func (db *PostgresDB) addJob(jid, status, mode, host, processID, submitter string, updated time.Time) error { - query := `INSERT INTO jobs (id, status, updated, mode, host, process_id, submitter) VALUES ($1, $2, $3, $4, $5, $6, $7)` - _, err := db.Handle.Exec(query, jid, status, updated, mode, host, processID, submitter) +func (db *PostgresDB) addJob(jid, status, mode, host, processID, submitter string, tags []string, updated time.Time) error { + query := `INSERT INTO jobs (id, status, updated, mode, host, process_id, submitter, tags) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)` + _, err := db.Handle.Exec(query, jid, status, updated, mode, host, processID, submitter, pq.Array(tags)) return err } @@ -75,9 +75,11 @@ func (db *PostgresDB) updateJobRecord(jid, status string, now time.Time) error { // GetJob retrieves a job record by id func (db *PostgresDB) GetJob(jid string) (JobRecord, bool, error) { - query := `SELECT * FROM jobs WHERE id = $1` + query := `SELECT id, status, updated, mode, host, process_id, submitter, tags FROM jobs WHERE id = $1` var jr JobRecord - err := db.Handle.QueryRow(query, jid).Scan(&jr.JobID, &jr.Status, &jr.LastUpdate, &jr.Mode, &jr.Host, &jr.ProcessID, &jr.Submitter) + err := db.Handle.QueryRow(query, jid).Scan( + &jr.JobID, &jr.Status, &jr.LastUpdate, &jr.Mode, &jr.Host, &jr.ProcessID, &jr.Submitter, pq.Array(&jr.Tags), + ) if err != nil { if err == sql.ErrNoRows { return JobRecord{}, false, nil @@ -102,12 +104,11 @@ func (db *PostgresDB) CheckJobExist(jid string) (bool, error) { } // Assumes query parameters are valid -func (pgDB *PostgresDB) GetJobs(limit, offset int, processIDs, statuses, submitters []string) ([]JobRecord, error) { - baseQuery := `SELECT id, status, updated, process_id, submitter FROM jobs` +func (pgDB *PostgresDB) GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string) ([]JobRecord, error) { + baseQuery := `SELECT id, status, updated, process_id, submitter, tags FROM jobs` whereClauses := []string{} args := []interface{}{} - - argIndex := 1 // Start from 1 for PostgreSQL placeholders + argIndex := 1 if len(processIDs) > 0 { placeholders := make([]string, len(processIDs)) @@ -120,7 +121,6 @@ func (pgDB *PostgresDB) GetJobs(limit, offset int, processIDs, statuses, submitt args = append(args, pid) } } - if len(statuses) > 0 { placeholders := make([]string, len(statuses)) for i := range statuses { @@ -132,7 +132,6 @@ func (pgDB *PostgresDB) GetJobs(limit, offset int, processIDs, statuses, submitt args = append(args, st) } } - if len(submitters) > 0 { placeholders := make([]string, len(submitters)) for i := range submitters { @@ -144,26 +143,30 @@ func (pgDB *PostgresDB) GetJobs(limit, offset int, processIDs, statuses, submitt args = append(args, sb) } } + // Tag filtering: @> means "contains all of" (AND logic) + for _, tag := range tags { + whereClauses = append(whereClauses, fmt.Sprintf("array_to_string(tags, ',') ILIKE $%d", argIndex)) + argIndex++ + args = append(args, "%"+tag+"%") + } if len(whereClauses) > 0 { baseQuery += " WHERE " + strings.Join(whereClauses, " AND ") } - // Add limit and offset to the query and args query := baseQuery + fmt.Sprintf(" ORDER BY updated DESC LIMIT $%d OFFSET $%d", argIndex, argIndex+1) args = append(args, limit, offset) - res := []JobRecord{} - rows, err := pgDB.Handle.Query(query, args...) if err != nil { return nil, err } defer rows.Close() + res := []JobRecord{} for rows.Next() { var r JobRecord - if err := rows.Scan(&r.JobID, &r.Status, &r.LastUpdate, &r.ProcessID, &r.Submitter); err != nil { + if err := rows.Scan(&r.JobID, &r.Status, &r.LastUpdate, &r.ProcessID, &r.Submitter, pq.Array(&r.Tags)); err != nil { return nil, err } res = append(res, r) diff --git a/api/jobs/database_sqlite.go b/api/jobs/database_sqlite.go index 193a68d..447d200 100644 --- a/api/jobs/database_sqlite.go +++ b/api/jobs/database_sqlite.go @@ -50,6 +50,14 @@ func NewSQLiteDB(dbPath string) (*SQLiteDB, error) { return &db, nil } +func joinTags(tags []string) string { return strings.Join(tags, ",") } +func splitTags(s string) []string { + if s == "" { + return []string{} + } + return strings.Split(s, ",") +} + // Create tables in the database if they do not exist already func (sqliteDB *SQLiteDB) createTables() error { @@ -63,37 +71,35 @@ func (sqliteDB *SQLiteDB) createTables() error { // providing job-lists ordered by time queryJobs := ` - CREATE TABLE IF NOT EXISTS jobs ( - id TEXT PRIMARY KEY, - status TEXT NOT NULL, - updated TIMESTAMP NOT NULL, - mode TEXT NOT NULL, - host TEXT NOT NULL, - process_id TEXT NOT NULL, - submitter TEXT NOT NULL DEFAULT '' - ); +CREATE TABLE IF NOT EXISTS jobs ( + id TEXT PRIMARY KEY, + status TEXT NOT NULL, + updated TIMESTAMP NOT NULL, + mode TEXT NOT NULL, + host TEXT NOT NULL, + process_id TEXT NOT NULL, + submitter TEXT NOT NULL DEFAULT '', + tags TEXT NOT NULL DEFAULT '' +); CREATE INDEX IF NOT EXISTS idx_jobs_updated ON jobs(updated); CREATE INDEX IF NOT EXISTS idx_jobs_process_id ON jobs(process_id); CREATE INDEX IF NOT EXISTS idx_jobs_submitter ON jobs(submitter); - ` +` _, err := sqliteDB.Handle.Exec(queryJobs) if err != nil { return fmt.Errorf("error creating tables: %s", err) } + return nil } // Add job to the database. Will return error if job exist. -func (sqliteDB *SQLiteDB) addJob(jid, status, mode, host, processID, submitter string, updated time.Time) error { - query := `INSERT INTO jobs (id, status, updated, mode, host, process_id, submitter) VALUES (?, ?, ?, ?, ?, ?, ?)` - - _, err := sqliteDB.Handle.Exec(query, jid, status, updated, mode, host, processID, submitter) - if err != nil { - return err - } - return nil +func (sqliteDB *SQLiteDB) addJob(jid, status, mode, host, processID, submitter string, tags []string, updated time.Time) error { + query := `INSERT INTO jobs (id, status, updated, mode, host, process_id, submitter, tags) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + _, err := sqliteDB.Handle.Exec(query, jid, status, updated, mode, host, processID, submitter, joinTags(tags)) + return err } // Update status and time of a job. @@ -110,20 +116,19 @@ func (sqliteDB *SQLiteDB) updateJobRecord(jid, status string, now time.Time) err // If job do not exists, or error encountered bool would be false. // Similar behavior as key exist in hashmap. func (sqliteDB *SQLiteDB) GetJob(jid string) (JobRecord, bool, error) { - query := `SELECT * FROM jobs WHERE id = ?` - + query := `SELECT id, status, updated, mode, host, process_id, submitter, tags FROM jobs WHERE id = ?` jr := JobRecord{} - + var tagsStr string row := sqliteDB.Handle.QueryRow(query, jid) - err := row.Scan(&jr.JobID, &jr.Status, &jr.LastUpdate, &jr.Mode, &jr.Host, &jr.ProcessID, &jr.Submitter) + err := row.Scan(&jr.JobID, &jr.Status, &jr.LastUpdate, &jr.Mode, &jr.Host, &jr.ProcessID, &jr.Submitter, &tagsStr) if err != nil { if err == sql.ErrNoRows { return JobRecord{}, false, nil - } else { - log.Error(err) - return JobRecord{}, false, err } + log.Error(err) + return JobRecord{}, false, err } + jr.Tags = splitTags(tagsStr) return jr, true, nil } @@ -146,8 +151,8 @@ func (sqliteDB *SQLiteDB) CheckJobExist(jid string) (bool, error) { } // Assumes query parameters are valid -func (sqliteDB *SQLiteDB) GetJobs(limit, offset int, processIDs, statuses []string, submitters []string) ([]JobRecord, error) { - baseQuery := `SELECT id, status, updated, process_id, submitter FROM jobs` +func (sqliteDB *SQLiteDB) GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string) ([]JobRecord, error) { + baseQuery := `SELECT id, status, updated, process_id, submitter, tags FROM jobs` whereClauses := []string{} args := []interface{}{} @@ -158,7 +163,6 @@ func (sqliteDB *SQLiteDB) GetJobs(limit, offset int, processIDs, statuses []stri args = append(args, pid) } } - if len(statuses) > 0 { placeholders := strings.Repeat("?,", len(statuses)-1) + "?" whereClauses = append(whereClauses, fmt.Sprintf("status IN (%s)", placeholders)) @@ -166,7 +170,6 @@ func (sqliteDB *SQLiteDB) GetJobs(limit, offset int, processIDs, statuses []stri args = append(args, st) } } - if len(submitters) > 0 { placeholders := strings.Repeat("?,", len(submitters)-1) + "?" whereClauses = append(whereClauses, fmt.Sprintf("submitter IN (%s)", placeholders)) @@ -174,6 +177,11 @@ func (sqliteDB *SQLiteDB) GetJobs(limit, offset int, processIDs, statuses []stri args = append(args, sb) } } + // Tag filtering: job must contain ALL requested tags (AND logic) + for _, tag := range tags { + whereClauses = append(whereClauses, `(',' || LOWER(tags) || ',') LIKE LOWER(?)`) + args = append(args, "%,"+strings.ToLower(tag)+",%") + } if len(whereClauses) > 0 { baseQuery += " WHERE " + strings.Join(whereClauses, " AND ") @@ -192,9 +200,11 @@ func (sqliteDB *SQLiteDB) GetJobs(limit, offset int, processIDs, statuses []stri for rows.Next() { var r JobRecord - if err := rows.Scan(&r.JobID, &r.Status, &r.LastUpdate, &r.ProcessID, &r.Submitter); err != nil { + var tagsStr string + if err := rows.Scan(&r.JobID, &r.Status, &r.LastUpdate, &r.ProcessID, &r.Submitter, &tagsStr); err != nil { return nil, err } + r.Tags = splitTags(tagsStr) res = append(res, r) } diff --git a/api/jobs/docker_jobs.go b/api/jobs/docker_jobs.go index 88763be..9d2608b 100644 --- a/api/jobs/docker_jobs.go +++ b/api/jobs/docker_jobs.go @@ -34,10 +34,10 @@ type DockerJob struct { Volumes []string `json:"volumes"` Cmd []string `json:"commandOverride"` UpdateTime time.Time - Status string `json:"status"` - - logger *log.Logger - logFile *os.File + Status string `json:"status"` + Tags []string `json:"tags"` + logger *log.Logger + logFile *os.File Resources DB Database @@ -214,7 +214,7 @@ func (j *DockerJob) Create() error { j.ctxCancel = cancelFunc // At this point job is ready to be added to database - err = j.DB.addJob(j.UUID, "accepted", "", "local", j.ProcessName, j.Submitter, time.Now()) + err = j.DB.addJob(j.UUID, "accepted", "", "local", j.ProcessName, j.Submitter, j.Tags, time.Now()) if err != nil { j.ctxCancel() return err diff --git a/api/jobs/jobs.go b/api/jobs/jobs.go index b219958..4413efe 100644 --- a/api/jobs/jobs.go +++ b/api/jobs/jobs.go @@ -77,6 +77,7 @@ type JobRecord struct { Host string `json:"host,omitempty"` Mode string `json:"mode,omitempty"` Submitter string `json:"submitter"` + Tags []string `json:"tags,omitempty"` } type LogEntry struct { diff --git a/api/jobs/subprocess_jobs.go b/api/jobs/subprocess_jobs.go index 16a7bcc..0f40d4c 100644 --- a/api/jobs/subprocess_jobs.go +++ b/api/jobs/subprocess_jobs.go @@ -31,9 +31,9 @@ type SubprocessJob struct { EnvVars []string Cmd []string `json:"commandOverride"` UpdateTime time.Time - Status string `json:"status"` - - execCmd *exec.Cmd + Status string `json:"status"` + Tags []string `json:"tags"` + execCmd *exec.Cmd logger *log.Logger logFile *os.File @@ -164,7 +164,8 @@ func (j *SubprocessJob) Create() error { j.ctxCancel = cancelFunc // At this point job is ready to be added to database - err = j.DB.addJob(j.UUID, "accepted", "", "local", j.ProcessName, j.Submitter, time.Now()) + err = j.DB.addJob(j.UUID, "accepted", "", "local", j.ProcessName, j.Submitter, j.Tags, time.Now()) + if err != nil { j.ctxCancel() return err diff --git a/api/views/jobs.html b/api/views/jobs.html index c98dd1d..1864cec 100644 --- a/api/views/jobs.html +++ b/api/views/jobs.html @@ -4,61 +4,64 @@ - - - Jobs List - + + + Jobs List + + -

Jobs List

- - - - - - - - - - - - {{range .jobs}} - - - - - - - - {{end}} - -
Logs (JobID)StatusProcessIDSubmitterUpdated
{{.JobID}} - {{if eq .Status "successful"}} - successful - {{else if eq .Status "failed"}} - failed - {{else if eq .Status "dismissed"}} - dismissed - {{else if or (eq .Status "accepted") (eq .Status "running")}} - in progress - {{end}} - - {{.Status}} - - {{.ProcessID}}{{.Submitter}}{{.LastUpdate.Format "2006-01-02 15:04:05 MST"}}
-
- +

Jobs List

+ + + + + + + + + + + + + {{range .jobs}} + + + + + + + + + {{end}} + +
Logs (JobID)StatusProcessIDTagsSubmitterUpdated
{{.JobID}} + {{if eq .Status "successful"}} + successful + {{else if eq .Status "failed"}} + failed + {{else if eq .Status "dismissed"}} + dismissed + {{else if or (eq .Status "accepted") (eq .Status "running")}} + in progress + {{end}} + + {{.Status}} + + {{.ProcessID}}{{range .Tags}}{{.}} {{end}}{{.Submitter}}{{.LastUpdate.Format "2006-01-02 15:04:05 MST"}}
+
+ -{{end}} \ No newline at end of file +{{end}} From 30ece959ea3958718cf6e43fc1d8d2d1ef74304d Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Tue, 3 Mar 2026 15:23:27 -0500 Subject: [PATCH 02/19] add tags check --- api/handlers/handlers.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/handlers/handlers.go b/api/handlers/handlers.go index 06fec3e..ace8fb9 100644 --- a/api/handlers/handlers.go +++ b/api/handlers/handlers.go @@ -191,6 +191,9 @@ func (rh *RESTHandler) Execution(c echo.Context) error { if params.Inputs == nil { return c.JSON(http.StatusBadRequest, errResponse{Message: "'inputs' is required in the body of the request"}) } + if params.Tags == nil { + return c.JSON(http.StatusBadRequest, errResponse{Message: "'Tags' is required in the body of the request"}) + } err = p.VerifyInputs(params.Inputs) if err != nil { From dfdd037ab59f3af6077bdd12ac8d3e0cbeb92d8b Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Thu, 5 Mar 2026 06:40:49 -0500 Subject: [PATCH 03/19] make tags a non required field and always returned in the get struct --- api/handlers/handlers.go | 4 ++-- api/jobs/jobs.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/handlers/handlers.go b/api/handlers/handlers.go index ace8fb9..c78b3dd 100644 --- a/api/handlers/handlers.go +++ b/api/handlers/handlers.go @@ -97,7 +97,7 @@ func prepareResponse(c echo.Context, httpStatus int, renderName string, output i // specs: https://developer.ogc.org/api/processes/index.html#tag/Execute type runRequestBody struct { Inputs map[string]interface{} `json:"inputs"` - Tags []string `json:"tags"` + Tags []string `json:"tags,omitempty"` } // LandingPage godoc @@ -192,7 +192,7 @@ func (rh *RESTHandler) Execution(c echo.Context) error { return c.JSON(http.StatusBadRequest, errResponse{Message: "'inputs' is required in the body of the request"}) } if params.Tags == nil { - return c.JSON(http.StatusBadRequest, errResponse{Message: "'Tags' is required in the body of the request"}) + params.Tags = []string{} // default to empty if not provided } err = p.VerifyInputs(params.Inputs) diff --git a/api/jobs/jobs.go b/api/jobs/jobs.go index 4413efe..642a9e8 100644 --- a/api/jobs/jobs.go +++ b/api/jobs/jobs.go @@ -77,7 +77,7 @@ type JobRecord struct { Host string `json:"host,omitempty"` Mode string `json:"mode,omitempty"` Submitter string `json:"submitter"` - Tags []string `json:"tags,omitempty"` + Tags []string `json:"tags"` } type LogEntry struct { From 2b111aacb895ab56e135a70f5dd87af426d71cd6 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Thu, 5 Mar 2026 07:10:56 -0500 Subject: [PATCH 04/19] add macID --- api/handlers/handlers.go | 7 ++++++- api/jobs/aws_batch_jobs.go | 3 ++- api/jobs/database.go | 4 ++-- api/jobs/database_postgres.go | 19 ++++++++++--------- api/jobs/database_sqlite.go | 19 ++++++++++--------- api/jobs/docker_jobs.go | 3 ++- api/jobs/jobs.go | 1 + api/jobs/subprocess_jobs.go | 3 ++- api/views/jobs.html | 2 ++ go.work.sum | 2 ++ 10 files changed, 39 insertions(+), 24 deletions(-) diff --git a/api/handlers/handlers.go b/api/handlers/handlers.go index c78b3dd..1e82bb7 100644 --- a/api/handlers/handlers.go +++ b/api/handlers/handlers.go @@ -98,6 +98,7 @@ func prepareResponse(c echo.Context, httpStatus int, renderName string, output i type runRequestBody struct { Inputs map[string]interface{} `json:"inputs"` Tags []string `json:"tags,omitempty"` + MacID string `json:"macID,omitempty"` } // LandingPage godoc @@ -247,6 +248,7 @@ func (rh *RESTHandler) Execution(c echo.Context) error { DB: rh.DB, DoneChan: rh.MessageQueue.JobDone, Tags: params.Tags, + MacID: params.MacID, } case "aws-batch": @@ -265,6 +267,7 @@ func (rh *RESTHandler) Execution(c echo.Context) error { DB: rh.DB, DoneChan: rh.MessageQueue.JobDone, Tags: params.Tags, + MacID: params.MacID, } case "subprocess": @@ -279,6 +282,7 @@ func (rh *RESTHandler) Execution(c echo.Context) error { DB: rh.DB, DoneChan: rh.MessageQueue.JobDone, Tags: params.Tags, + MacID: params.MacID, } } @@ -586,6 +590,7 @@ func (rh *RESTHandler) ListJobsHandler(c echo.Context) error { if tagsParam != "" { tagsList = strings.Split(tagsParam, ",") } + macIDParam := c.QueryParam("macID") var processIDList []string if processIDs != "" { @@ -629,7 +634,7 @@ func (rh *RESTHandler) ListJobsHandler(c echo.Context) error { offset = 0 } - result, err := rh.DB.GetJobs(limit, offset, processIDList, statusList, submittersList, tagsList) + result, err := rh.DB.GetJobs(limit, offset, processIDList, statusList, submittersList, tagsList, macIDParam) if err != nil { output := errResponse{HTTPStatus: http.StatusInternalServerError, Message: err.Error()} return prepareResponse(c, http.StatusNotFound, "error", output) diff --git a/api/jobs/aws_batch_jobs.go b/api/jobs/aws_batch_jobs.go index ac6fc4b..76d4348 100644 --- a/api/jobs/aws_batch_jobs.go +++ b/api/jobs/aws_batch_jobs.go @@ -39,6 +39,7 @@ type AWSBatchJob struct { Status string `json:"status"` // results interface{} Tags []string `json:"tags"` + MacID string `json:"macID"` logger *log.Logger logFile *os.File @@ -251,7 +252,7 @@ func (j *AWSBatchJob) Create() error { j.batchContext = batchContext // At this point job is ready to be added to database - err = j.DB.addJob(j.UUID, "accepted", "", "aws-batch", j.ProcessName, j.Submitter, j.Tags, time.Now()) + err = j.DB.addJob(j.UUID, "accepted", "", "aws-batch", j.ProcessName, j.Submitter, j.Tags, j.MacID, time.Now()) if err != nil { j.ctxCancel() return err diff --git a/api/jobs/database.go b/api/jobs/database.go index 3a24e5e..c78e53b 100644 --- a/api/jobs/database.go +++ b/api/jobs/database.go @@ -8,11 +8,11 @@ import ( // Database interface abstracts database operations type Database interface { - addJob(jid, status, mode, host, processID, submitter string, tags []string, updated time.Time) error + addJob(jid, status, mode, host, processID, submitter string, tags []string, macID string, updated time.Time) error updateJobRecord(jid, status string, now time.Time) error GetJob(jid string) (JobRecord, bool, error) CheckJobExist(jid string) (bool, error) - GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string) ([]JobRecord, error) + GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string, macID string) ([]JobRecord, error) Close() error } diff --git a/api/jobs/database_postgres.go b/api/jobs/database_postgres.go index 961cc19..f462e15 100644 --- a/api/jobs/database_postgres.go +++ b/api/jobs/database_postgres.go @@ -46,7 +46,8 @@ func (postgresDB *PostgresDB) createTables() error { host TEXT NOT NULL, process_id TEXT NOT NULL, submitter TEXT NOT NULL DEFAULT '', - tags TEXT[] NOT NULL DEFAULT '{}' + tags TEXT[] NOT NULL DEFAULT '{}', + macID TEXT NOT NULL DEFAULT '' ); CREATE INDEX IF NOT EXISTS idx_jobs_updated ON jobs(updated); CREATE INDEX IF NOT EXISTS idx_jobs_process_id ON jobs(process_id); @@ -60,9 +61,9 @@ func (postgresDB *PostgresDB) createTables() error { } // AddJob adds a new job to the database -func (db *PostgresDB) addJob(jid, status, mode, host, processID, submitter string, tags []string, updated time.Time) error { - query := `INSERT INTO jobs (id, status, updated, mode, host, process_id, submitter, tags) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)` - _, err := db.Handle.Exec(query, jid, status, updated, mode, host, processID, submitter, pq.Array(tags)) +func (db *PostgresDB) addJob(jid, status, mode, host, processID, submitter string, tags []string, macID string, updated time.Time) error { + query := `INSERT INTO jobs (id, status, updated, mode, host, process_id, submitter, tags, macID) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)` + _, err := db.Handle.Exec(query, jid, status, updated, mode, host, processID, submitter, pq.Array(tags), macID) return err } @@ -75,10 +76,10 @@ func (db *PostgresDB) updateJobRecord(jid, status string, now time.Time) error { // GetJob retrieves a job record by id func (db *PostgresDB) GetJob(jid string) (JobRecord, bool, error) { - query := `SELECT id, status, updated, mode, host, process_id, submitter, tags FROM jobs WHERE id = $1` + query := `SELECT id, status, updated, mode, host, process_id, submitter, tags, macID FROM jobs WHERE id = $1` var jr JobRecord err := db.Handle.QueryRow(query, jid).Scan( - &jr.JobID, &jr.Status, &jr.LastUpdate, &jr.Mode, &jr.Host, &jr.ProcessID, &jr.Submitter, pq.Array(&jr.Tags), + &jr.JobID, &jr.Status, &jr.LastUpdate, &jr.Mode, &jr.Host, &jr.ProcessID, &jr.Submitter, pq.Array(&jr.Tags), &jr.MacID, ) if err != nil { if err == sql.ErrNoRows { @@ -104,8 +105,8 @@ func (db *PostgresDB) CheckJobExist(jid string) (bool, error) { } // Assumes query parameters are valid -func (pgDB *PostgresDB) GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string) ([]JobRecord, error) { - baseQuery := `SELECT id, status, updated, process_id, submitter, tags FROM jobs` +func (pgDB *PostgresDB) GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string, macID string) ([]JobRecord, error) { + baseQuery := `SELECT id, status, updated, process_id, submitter, tags, macID FROM jobs` whereClauses := []string{} args := []interface{}{} argIndex := 1 @@ -166,7 +167,7 @@ func (pgDB *PostgresDB) GetJobs(limit, offset int, processIDs, statuses, submitt res := []JobRecord{} for rows.Next() { var r JobRecord - if err := rows.Scan(&r.JobID, &r.Status, &r.LastUpdate, &r.ProcessID, &r.Submitter, pq.Array(&r.Tags)); err != nil { + if err := rows.Scan(&r.JobID, &r.Status, &r.LastUpdate, &r.ProcessID, &r.Submitter, pq.Array(&r.Tags), &r.MacID); err != nil { return nil, err } res = append(res, r) diff --git a/api/jobs/database_sqlite.go b/api/jobs/database_sqlite.go index 447d200..f2ecd6b 100644 --- a/api/jobs/database_sqlite.go +++ b/api/jobs/database_sqlite.go @@ -79,7 +79,8 @@ CREATE TABLE IF NOT EXISTS jobs ( host TEXT NOT NULL, process_id TEXT NOT NULL, submitter TEXT NOT NULL DEFAULT '', - tags TEXT NOT NULL DEFAULT '' + tags TEXT NOT NULL DEFAULT '', + macID TEXT NOT NULL DEFAULT '' ); CREATE INDEX IF NOT EXISTS idx_jobs_updated ON jobs(updated); @@ -96,9 +97,9 @@ CREATE TABLE IF NOT EXISTS jobs ( } // Add job to the database. Will return error if job exist. -func (sqliteDB *SQLiteDB) addJob(jid, status, mode, host, processID, submitter string, tags []string, updated time.Time) error { - query := `INSERT INTO jobs (id, status, updated, mode, host, process_id, submitter, tags) VALUES (?, ?, ?, ?, ?, ?, ?, ?)` - _, err := sqliteDB.Handle.Exec(query, jid, status, updated, mode, host, processID, submitter, joinTags(tags)) +func (sqliteDB *SQLiteDB) addJob(jid, status, mode, host, processID, submitter string, tags []string, macID string, updated time.Time) error { + query := `INSERT INTO jobs (id, status, updated, mode, host, process_id, submitter, tags, macID) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + _, err := sqliteDB.Handle.Exec(query, jid, status, updated, mode, host, processID, submitter, joinTags(tags), macID) return err } @@ -116,11 +117,11 @@ func (sqliteDB *SQLiteDB) updateJobRecord(jid, status string, now time.Time) err // If job do not exists, or error encountered bool would be false. // Similar behavior as key exist in hashmap. func (sqliteDB *SQLiteDB) GetJob(jid string) (JobRecord, bool, error) { - query := `SELECT id, status, updated, mode, host, process_id, submitter, tags FROM jobs WHERE id = ?` + query := `SELECT id, status, updated, mode, host, process_id, submitter, tags, macID FROM jobs WHERE id = ?` jr := JobRecord{} var tagsStr string row := sqliteDB.Handle.QueryRow(query, jid) - err := row.Scan(&jr.JobID, &jr.Status, &jr.LastUpdate, &jr.Mode, &jr.Host, &jr.ProcessID, &jr.Submitter, &tagsStr) + err := row.Scan(&jr.JobID, &jr.Status, &jr.LastUpdate, &jr.Mode, &jr.Host, &jr.ProcessID, &jr.Submitter, &tagsStr, &jr.MacID) if err != nil { if err == sql.ErrNoRows { return JobRecord{}, false, nil @@ -151,8 +152,8 @@ func (sqliteDB *SQLiteDB) CheckJobExist(jid string) (bool, error) { } // Assumes query parameters are valid -func (sqliteDB *SQLiteDB) GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string) ([]JobRecord, error) { - baseQuery := `SELECT id, status, updated, process_id, submitter, tags FROM jobs` +func (sqliteDB *SQLiteDB) GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string, macID string) ([]JobRecord, error) { + baseQuery := `SELECT id, status, updated, process_id, submitter, tags, macID FROM jobs` whereClauses := []string{} args := []interface{}{} @@ -201,7 +202,7 @@ func (sqliteDB *SQLiteDB) GetJobs(limit, offset int, processIDs, statuses, submi for rows.Next() { var r JobRecord var tagsStr string - if err := rows.Scan(&r.JobID, &r.Status, &r.LastUpdate, &r.ProcessID, &r.Submitter, &tagsStr); err != nil { + if err := rows.Scan(&r.JobID, &r.Status, &r.LastUpdate, &r.ProcessID, &r.Submitter, &tagsStr, &r.MacID); err != nil { return nil, err } r.Tags = splitTags(tagsStr) diff --git a/api/jobs/docker_jobs.go b/api/jobs/docker_jobs.go index 9d2608b..03bde82 100644 --- a/api/jobs/docker_jobs.go +++ b/api/jobs/docker_jobs.go @@ -38,6 +38,7 @@ type DockerJob struct { Tags []string `json:"tags"` logger *log.Logger logFile *os.File + MacID string `json:"macID"` Resources DB Database @@ -214,7 +215,7 @@ func (j *DockerJob) Create() error { j.ctxCancel = cancelFunc // At this point job is ready to be added to database - err = j.DB.addJob(j.UUID, "accepted", "", "local", j.ProcessName, j.Submitter, j.Tags, time.Now()) + err = j.DB.addJob(j.UUID, "accepted", "", "local", j.ProcessName, j.Submitter, j.Tags, j.MacID, time.Now()) if err != nil { j.ctxCancel() return err diff --git a/api/jobs/jobs.go b/api/jobs/jobs.go index 642a9e8..e4d133f 100644 --- a/api/jobs/jobs.go +++ b/api/jobs/jobs.go @@ -78,6 +78,7 @@ type JobRecord struct { Mode string `json:"mode,omitempty"` Submitter string `json:"submitter"` Tags []string `json:"tags"` + MacID string `json:"macID"` } type LogEntry struct { diff --git a/api/jobs/subprocess_jobs.go b/api/jobs/subprocess_jobs.go index 0f40d4c..9d44ac8 100644 --- a/api/jobs/subprocess_jobs.go +++ b/api/jobs/subprocess_jobs.go @@ -33,6 +33,7 @@ type SubprocessJob struct { UpdateTime time.Time Status string `json:"status"` Tags []string `json:"tags"` + MacID string `json:"macID"` execCmd *exec.Cmd logger *log.Logger @@ -164,7 +165,7 @@ func (j *SubprocessJob) Create() error { j.ctxCancel = cancelFunc // At this point job is ready to be added to database - err = j.DB.addJob(j.UUID, "accepted", "", "local", j.ProcessName, j.Submitter, j.Tags, time.Now()) + err = j.DB.addJob(j.UUID, "accepted", "", "local", j.ProcessName, j.Submitter, j.Tags, j.MacID, time.Now()) if err != nil { j.ctxCancel() diff --git a/api/views/jobs.html b/api/views/jobs.html index 1864cec..71e836c 100644 --- a/api/views/jobs.html +++ b/api/views/jobs.html @@ -20,6 +20,7 @@

Jobs List

Status ProcessID Tags + MacID Submitter Updated @@ -44,6 +45,7 @@

Jobs List

{{.ProcessID}} {{range .Tags}}{{.}} {{end}} + {{.MacID}} {{.Submitter}} {{.LastUpdate.Format "2006-01-02 15:04:05 MST"}} diff --git a/go.work.sum b/go.work.sum index cb7f239..56438a3 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1,10 +1,12 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= From 61a2675374118c477ead5afe4303d608d0dc56eb Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Thu, 5 Mar 2026 07:23:26 -0500 Subject: [PATCH 05/19] fix target sql query --- api/jobs/database_postgres.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/jobs/database_postgres.go b/api/jobs/database_postgres.go index 9caed5f..2b80098 100644 --- a/api/jobs/database_postgres.go +++ b/api/jobs/database_postgres.go @@ -75,7 +75,7 @@ func (postgresDB *PostgresDB) createTables() error { // AddJob adds a new job to the database func (db *PostgresDB) addJob(jid, status, mode, host, hostJobID, processID, submitter string, tags []string, macID string, updated time.Time) error { - query := `INSERT INTO jobs (id, status, updated, mode, host, host_job_id, process_id, submitter, tags, macID) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $8, $9)` + query := `INSERT INTO jobs (id, status, updated, mode, host, host_job_id, process_id, submitter, tags, macID) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)` _, err := db.Handle.Exec(query, jid, status, updated, mode, host, hostJobID, processID, submitter, pq.Array(tags), macID) return err } From efca711b7304030df9a072e12fd74fe0e0b42a1d Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Thu, 5 Mar 2026 07:32:15 -0500 Subject: [PATCH 06/19] add changes to change and dev log --- CHANGELOG.md | 34 ++++++++++++++++++++++++++++++++-- DEV_GUIDE.md | 32 ++++++++++++++++++++++++-------- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fc2ad9..ca916c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,53 +9,83 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Unreleased +## [0.2.3] - 2025-2-28 + +### API + +#### POST /processes/:processID/execution + +- Added optional `tags` array to execution payload. Tags are stored with the job record and can be used to filter jobs when querying `/jobs`. +- Added optional `macID` string to execution payload to associate jobs with a specific client or machine. + +#### GET /jobs + +- Added `macID` query parameter to filter jobs by machine identifier. +- Job responses now include `tags` and `macID` fields. + ## [0.2.2] - 2025-2-28 ### API + #### POST /processes/:processID/execution + - Execution mode now determined per OGC API - Processes Requirements 25/26: honors `Prefer: respond-async` header when process supports both modes, defaults to sync otherwise - Returns `Preference-Applied` response header when async preference is honored #### GET /admin/resources + - New endpoint to view resource utilization for local jobs (docker, subprocess) and queue status + #### GET /jobs/:jobID/metadata + - Added `recoveryNotice` object to metadata format ### Features + - Added restart recovery for docker, subprocess, and AWS Batch jobs, including status reconciliation and log handling. - Introduced `lost` job status and surfaced it in UI status indicators (job list, job logs, status table). - Recovered jobs annotate metadata with a recovery notice and write best-effort metadata when some fields are missing. - Resource pool can force-reserve resources for already-running recovered docker jobs. ### Configuration + - New `MAX_LOCAL_CPUS` and `MAX_LOCAL_MEMORY` environment variables (or `--max-local-cpus` and `--max-local-memory` CLI flags) to set resource limits for local job scheduling - Process definitions are validated against these limits at startup and when adding/updating processes via API - Processes without explicit resource requirements use default values ### Documentation + - Added sequence diagram for local scheduler - Added Recovery section to DEV_GUIDE ## [0.2.1] - 2025-12-03 ### API + - Version information is added in landing page. ### Configuration + - Repository URL is now configurable via `REPO_URL` environment variable. This URL is used for version links and metadata context references. ### Documentation + - Changelog updated to new format ## [0.2.0] - 2025-12-02 ### API + #### GET /jobs/:jobID/logs + - In response body `container_logs` key is replaced by `process_logs` + #### PUT|POST /processes/:processID + - Request payload schema has changed (See Process YAML Schema changes below) ### Process YAML Schema + - `command` is now a first class object and moved outside of `container` - `config` object is added - `maxResources` and `envVars` are moved under `config` object @@ -64,14 +94,15 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - `host.type` valid options are changed from `local` | `aws-batch` to `docker` | `aws-batch` | `subprocess` ### Features + - `subprocess` type processes now can be executed through API. They must be registered like other processes and will be called using OS subprocess calls. ### Documentation + - A `CHANGELOG.md` file is added in the repo. - Process templates are provided for all three host types in `./process_templates` folder - Windows setup instructions are added in `README.md` - ## [0.1.0] - 2023-07-07 - Initial release with core API endpoints for process and job management @@ -79,6 +110,5 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm [Unreleased]: https://github.com/Dewberry/sepex/compare/v0.2.2...HEAD [0.2.2]: https://github.com/Dewberry/sepex/releases/tag/v0.2.2 [0.2.1]: https://github.com/Dewberry/sepex/releases/tag/v0.2.1 - [0.2.0]: https://github.com/Dewberry/sepex/releases/tag/v0.2.0 [0.1.0]: https://github.com/Dewberry/sepex/releases/tag/v0.1.0 diff --git a/DEV_GUIDE.md b/DEV_GUIDE.md index 10f4ff1..94766eb 100644 --- a/DEV_GUIDE.md +++ b/DEV_GUIDE.md @@ -1,19 +1,22 @@ ## Config + - All secrets and configuration settings are handled through environment variables - There is an example.env provided to ease the configuration process - Command line flags are available for config that is only needed at startup, they take precedence over the environment variables when used. - Other configs are defined through env variables so that they can be modified without restarting the server. - Here is the resolution order: - - Flag, where option is available and used - - Environment variable - - Default value, where available + - Flag, where option is available and used + - Environment variable + - Default value, where available ## Process Specific Env + - They must start with ALL CAPS process id. - They will be passed to jobs with process id prefix removed. This allow setting 3rd party env variables such as GDAL_NUM_CPUS etc. - We are parsing at the job level so as to allow dynamic updates without having to restart server ## Auth + - If auth is enabled some or all routes are protected based on env variable `AUTH_LEVEL` settings. - The middleware validate and parse JWT to verify `X-SEPEX-User-Email` header and inject `X-SEPEX-User-Roles` header. - A user can use tools like Postman to set these headers themselves, but if auth is enabled, they will be checked against the token. This setup allows adding submitter info to the database when auth is not enabled. @@ -25,9 +28,25 @@ - Only admins can add/update/delete processes. ## Inputs -- If `"Inputs": {}` in `/execution` payload. Nothing will be appended to process commands. This allow running processes that do not have any inputs. + +- If `"inputs": {}` in `/execution` payload, nothing will be appended to process commands. This allows running processes that do not have any inputs. +- `"tags"` (optional) can be included as an array of strings in the `/execution` payload. Tags are stored with the job record and can be used to filter jobs when querying `/jobs`. If omitted, the job will be created with an empty tag list. +- `"macID"` (optional) can be included as a string in the `/execution` payload. This value is stored with the job record and can be used to associate jobs with a specific machine or client and filter jobs via the `/jobs` endpoint. + +Example payload: + +```JSON +{ + "inputs": { + "text": "Hello World" + }, + "tags": ["example", "test"], + "macID": "machine-01" +} +``` ## Scope + - The behavior of logging is unknown for AWS Batch processes with job definitions having number of attempts more than 1. ## Local Scheduler @@ -54,7 +73,6 @@ Recovery rebuilds in-memory state after a restart using DB non-terminal jobs. 1. Metadata for recovered jobs will be incomplete but present. - ## Release/Versioning/Changelog The project uses an automated release workflow triggered by semver tags (e.g., `v1.0.0`, `v1.0.0-beta`). The workflow validates prerequisites, runs security scans, builds multi-platform container images, and creates GitHub releases with auto-generated release notes. @@ -68,6 +86,7 @@ The project uses an automated release workflow triggered by semver tags (e.g., ` - Release workflow fails if version is missing from CHANGELOG.md 2. **Create and Push a Semver Tag** + ```bash # For a regular release git tag v1.0.0 @@ -91,6 +110,3 @@ The project uses an automated release workflow triggered by semver tags (e.g., ` - Go to Actions tab → Release workflow → Run workflow - Select the tag from the dropdown - This is useful for re-running a release if needed - - - From de6e9b5b0419728e83753770b5612066ba707f0f Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Thu, 5 Mar 2026 08:10:44 -0500 Subject: [PATCH 07/19] fix minor errors --- CHANGELOG.md | 2 +- api/jobs/database_postgres.go | 8 +++++++- api/jobs/database_sqlite.go | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca916c9..67b50ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Unreleased -## [0.2.3] - 2025-2-28 +## [0.2.3] - 2026-3-05 ### API diff --git a/api/jobs/database_postgres.go b/api/jobs/database_postgres.go index 2b80098..2978ecb 100644 --- a/api/jobs/database_postgres.go +++ b/api/jobs/database_postgres.go @@ -67,9 +67,15 @@ func (postgresDB *PostgresDB) createTables() error { } // Backfill schema for older databases that predate host_job_id. - if _, err := postgresDB.Handle.Exec(`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS host_job_id TEXT NOT NULL DEFAULT ''`); err != nil { + if _, err := postgresDB.Handle.Exec(`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS host_job_id TEXT NOT NULL DEFAULT '';`); err != nil { return fmt.Errorf("error adding host_job_id column: %s", err) } + if _, err := postgresDB.Handle.Exec(`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS tags TEXT[] NOT NULL DEFAULT '{}';`); err != nil { + return fmt.Errorf("error adding tags column: %s", err) + } + if _, err := postgresDB.Handle.Exec(`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS macID TEXT NOT NULL DEFAULT '';`); err != nil { + return fmt.Errorf("error adding macID column: %s", err) + } return nil } diff --git a/api/jobs/database_sqlite.go b/api/jobs/database_sqlite.go index a2dfe83..23b474f 100644 --- a/api/jobs/database_sqlite.go +++ b/api/jobs/database_sqlite.go @@ -124,7 +124,7 @@ func (sqliteDB *SQLiteDB) updateJobRecord(jid, status string, now time.Time) err // If job do not exists, or error encountered bool would be false. // Similar behavior as key exist in hashmap. func (sqliteDB *SQLiteDB) GetJob(jid string) (JobRecord, bool, error) { - query := `SELECT id, status, updated, mode, host, process_id, submitter, tags, macID FROM jobs WHERE id = ?` + query := `SELECT id, status, updated, mode, host, host_job_id, process_id, submitter, tags, macID FROM jobs WHERE id = ?` jr := JobRecord{} var tagsStr string row := sqliteDB.Handle.QueryRow(query, jid) From 3980d2e39541b32a2141d8e877fce95ecdf3ce69 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Fri, 13 Mar 2026 10:25:30 -0400 Subject: [PATCH 08/19] add e2e tests for tags --- tests/e2e/tests.postman_collection.json | 6719 ++++++++++++----------- 1 file changed, 3552 insertions(+), 3167 deletions(-) diff --git a/tests/e2e/tests.postman_collection.json b/tests/e2e/tests.postman_collection.json index 1e37b17..8c85c87 100644 --- a/tests/e2e/tests.postman_collection.json +++ b/tests/e2e/tests.postman_collection.json @@ -1,3168 +1,3553 @@ { - "info": { - "_postman_id": "e4cfedee-5b49-464a-beba-6a8a9648e76e", - "name": "sepex Testing", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", - "_exporter_id": "36773789" - }, - "item": [ - { - "name": "admin", - "item": [ - { - "name": "resources", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Status code is 200', function () {", - " pm.response.to.have.status(200);", - "});", - "", - "pm.test('Response has resources object', function () {", - " var jsonData = pm.response.json();", - " pm.expect(jsonData).to.have.property('resources');", - "});", - "", - "pm.test('Response has links array', function () {", - " var jsonData = pm.response.json();", - " pm.expect(jsonData).to.have.property('links');", - " pm.expect(jsonData.links).to.be.an('array');", - "});", - "", - "pm.test('Resources has required fields', function () {", - " var resources = pm.response.json().resources;", - " pm.expect(resources).to.have.property('usedCPUs');", - " pm.expect(resources).to.have.property('usedMemory');", - " pm.expect(resources).to.have.property('queuedCPUs');", - " pm.expect(resources).to.have.property('queuedMemory');", - " pm.expect(resources).to.have.property('maxCPUs');", - " pm.expect(resources).to.have.property('maxMemory');", - " pm.expect(resources).to.have.property('usedCPUsPct');", - " pm.expect(resources).to.have.property('queuedCPUsPct');", - " pm.expect(resources).to.have.property('usedMemPct');", - " pm.expect(resources).to.have.property('queuedMemPct');", - "});", - "", - "pm.test('Percentage values are valid numbers', function () {", - " var resources = pm.response.json().resources;", - " pm.expect(resources.usedCPUsPct).to.be.a('number');", - " pm.expect(resources.usedMemPct).to.be.a('number');", - " pm.expect(resources.queuedCPUsPct).to.be.a('number');", - " pm.expect(resources.queuedMemPct).to.be.a('number');", - "});", - "", - "pm.test('Max values are greater than 0', function () {", - " var resources = pm.response.json().resources;", - " pm.expect(resources.maxCPUs).to.be.above(0);", - " pm.expect(resources.maxMemory).to.be.above(0);", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/admin/resources", - "host": [ - "{{url}}" - ], - "path": [ - "admin", - "resources" - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"Response time is less than 1000ms\", function () {", - " pm.expect(pm.response.responseTime).to.be.below(1000);", - "});" - ] - } - } - ] - }, - { - "name": "startup", - "item": [ - { - "name": "landingpage", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/", - "host": [ - "{{url}}" - ], - "path": [ - "" - ] - } - }, - "response": [] - }, - { - "name": "conformance", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/conformance", - "host": [ - "{{url}}" - ], - "path": [ - "conformance" - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('Status code is 200 ', function(){", - " pm.response.to.have.status(200)", - "});", - "", - "pm.test(\"Response time is less than 1000ms\", function () {", - " pm.expect(pm.response.responseTime).to.be.below(1000);", - "});", - "", - "if (pm.response.responseTime > 100) {", - " console.log(`⚠️ Warning: Response time is ${pm.response.responseTime}ms which is greater than 100ms.`);", - "}" - ] - } - } - ] - }, - { - "name": "processes", - "item": [ - { - "name": "processes", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "const processData = pm.response.json().processes;\r", - "\r", - "pm.test(\"number of processes registered test\", function () {\r", - " pm.expect(processData.length).to.eql(1);\r", - "});\r", - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/processes?limit=1&offset=1", - "host": [ - "{{url}}" - ], - "path": [ - "processes" - ], - "query": [ - { - "key": "limit", - "value": "1" - }, - { - "key": "offset", - "value": "1" - } - ] - } - }, - "response": [] - }, - { - "name": "process-describe", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "const processData = pm.response.json();\r", - "\r", - "pm.test(\"id test\", function () {\r", - " pm.expect(processData[\"info\"][\"id\"]).to.eql(\"pyecho\");\r", - "});\r", - "\r", - "pm.test(\"job control test\", function () {\r", - " pm.expect(processData[\"info\"][\"jobControlOptions\"][0]).to.eql(\"sync-execute\");\r", - "});" - ], - "type": "text/javascript", - "packages": {} - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/processes/:processID", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID" - ], - "variable": [ - { - "key": "processID", - "value": "pyecho" - } - ] - } - }, - "response": [] - }, - { - "name": "delete-startup-process", - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/processes/:processID", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID" - ], - "variable": [ - { - "key": "processID", - "value": "pywrite" - } - ] - } - }, - "response": [] - }, - { - "name": "add-process", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"info\": {\r\n \"version\": \"12.5.2023\",\r\n \"id\": \"dfcTest\",\r\n \"title\": \"Depth Frequency Curve\",\r\n \"description\": \"Returns data for depth-frequency curve\",\r\n \"jobControlOptions\": [\"async-execute\"],\r\n \"outputTransmission\": [\"value\"]\r\n },\r\n \"host\": {\r\n \"type\": \"aws-batch\",\r\n \"jobDefinition\": \"process-sandbox:2\",\r\n \"jobQueue\": \"micro-test\"\r\n },\r\n \"command\": [\"python\", \"dfc.py\"],\r\n \"inputs\": [\r\n {\r\n \"id\": \"crs\",\r\n \"title\": \"crs\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 0,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"tile\",\r\n \"title\": \"tile\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"epoch\",\r\n \"title\": \"epoch\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"points\",\r\n \"title\": \"points\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 999999\r\n }\r\n ],\r\n \"outputs\": [\r\n {\r\n \"id\": \"dfc\",\r\n \"title\": \"dfc\",\r\n \"description\": \"\",\r\n \"output\": {\r\n \"transmissionMode\": [\r\n \"value\"\r\n ]\r\n }\r\n }\r\n ]\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID" - ], - "variable": [ - { - "key": "processID", - "value": "dfcTest" - } - ] - } - }, - "response": [] - }, - { - "name": "update-process", - "request": { - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"info\": {\r\n \"version\": \"12.5.2023\",\r\n \"id\": \"dfcTest\",\r\n \"title\": \"Updated Depth Frequency Curve\",\r\n \"description\": \"Returns data for depth-frequency curve\",\r\n \"jobControlOptions\": [\"async-execute\"],\r\n \"outputTransmission\": [\"value\"]\r\n },\r\n \"host\": {\r\n \"type\": \"aws-batch\",\r\n \"jobDefinition\": \"process-sandbox:2\",\r\n \"jobQueue\": \"micro-test\"\r\n },\r\n \"command\": [\"python\", \"dfc.py\"],\r\n \"inputs\": [\r\n {\r\n \"id\": \"crs\",\r\n \"title\": \"crs\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 0,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"tile\",\r\n \"title\": \"tile\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"epoch\",\r\n \"title\": \"epoch\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"points\",\r\n \"title\": \"points\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 999999\r\n }\r\n ],\r\n \"outputs\": [\r\n {\r\n \"id\": \"dfc\",\r\n \"title\": \"dfc\",\r\n \"description\": \"\",\r\n \"output\": {\r\n \"transmissionMode\": [\r\n \"value\"\r\n ]\r\n }\r\n }\r\n ]\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID" - ], - "variable": [ - { - "key": "processID", - "value": "dfcTest" - } - ] - } - }, - "response": [] - }, - { - "name": "delete-process", - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/processes/:processID", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID" - ], - "variable": [ - { - "key": "processID", - "value": "dfcTest" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('Status code is 200 ', function(){", - " pm.response.to.have.status(200)", - "});", - "", - "pm.test(\"Response time is less than 1000ms\", function () {", - " pm.expect(pm.response.responseTime).to.be.below(1000);", - "});", - "", - "if (pm.response.responseTime > 100) {", - " console.log(`⚠️ Warning: Response time is ${pm.response.responseTime}ms which is greater than 100ms.`);", - "}" - ] - } - } - ] - }, - { - "name": "execution", - "item": [ - { - "name": "general", - "item": [ - { - "name": "Negatives", - "item": [ - { - "name": "missing", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {\n \"crs\": \"4326\",\n \"points\": [\n [\n -76.491824,\n 37.271065\n ]\n ],\n \"epoch\": \"2020\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "dfc" - } - ] - } - }, - "response": [] - }, - { - "name": "incorrectKey", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {\n \"incorrectKey\": \"value\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "pyecho" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"Status code is equal or greater than 400\", function () {", - " pm.expect(pm.response.code).to.be.greaterThan(399)", - "});", - "", - "pm.test(\"response should have message\", function () {", - " pm.response.to.have.jsonBody(\"message\");", - "});" - ] - } - } - ] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - } - ] - }, - { - "name": "sync-docker", - "item": [ - { - "name": "Positives", - "item": [ - { - "name": "job-submit", - "item": [ - { - "name": "pyecho", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {\n \"text\": \"Hello World!\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "pyecho" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "var resp = pm.response.json()", - "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" - ] - } - } - ] - }, - { - "name": "job-details", - "item": [ - { - "name": "job-status", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test(\"response should have jobid, status, etc\", function () {\r", - " pm.response.to.not.be.error;\r", - " pm.response.to.have.jsonBody(\"jobID\");\r", - " pm.response.to.have.jsonBody(\"status\");\r", - " pm.response.to.have.jsonBody(\"updated\");\r", - " pm.response.to.have.jsonBody(\"processID\");\r", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-logs", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/logs", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "logs" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-results", - "event": [ - { - "listen": "prerequest", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/results", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "results" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-metadata", - "event": [ - { - "listen": "prerequest", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/metadata", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "metadata" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - } - ] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('Status code is 200', function(){", - " pm.response.to.have.status(200)", - "});", - "", - "" - ] - } - } - ] - }, - { - "name": "Negatives", - "item": [ - { - "name": "job-dismiss", - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"Status code is equal or greater than 400\", function () {", - " pm.expect(pm.response.code).to.be.greaterThan(399)", - "});", - "", - "pm.test(\"response should have message\", function () {", - " pm.response.to.have.jsonBody(\"message\");", - "});" - ] - } - } - ] - } - ] - }, - { - "name": "sync-subprocess", - "item": [ - { - "name": "Positives", - "item": [ - { - "name": "job-submit", - "item": [ - { - "name": "subprocessEcho", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript", - "packages": {} - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "subprocessEcho" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "packages": {}, - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "packages": {}, - "exec": [ - "var resp = pm.response.json()\r", - "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" - ] - } - } - ] - }, - { - "name": "job-details", - "item": [ - { - "name": "job-status", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test(\"response should have jobid, status, etc\", function () {\r", - " pm.response.to.not.be.error;\r", - " pm.response.to.have.jsonBody(\"jobID\");\r", - " pm.response.to.have.jsonBody(\"status\");\r", - " pm.response.to.have.jsonBody(\"updated\");\r", - " pm.response.to.have.jsonBody(\"processID\");\r", - "});" - ], - "type": "text/javascript", - "packages": {} - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-logs", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/logs", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "logs" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-results", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/results", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "results" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-metadata", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/metadata", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "metadata" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - } - ] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "packages": {}, - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "packages": {}, - "exec": [ - "pm.test('Status code is 200', function(){\r", - " pm.response.to.have.status(200)\r", - "});\r", - "\r", - "" - ] - } - } - ] - }, - { - "name": "Negatives", - "item": [ - { - "name": "job-dismiss", - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "packages": {}, - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "packages": {}, - "exec": [ - "pm.test(\"Status code is equal or greater than 400\", function () {\r", - " pm.expect(pm.response.code).to.be.greaterThan(399)\r", - "});\r", - "\r", - "pm.test(\"response should have message\", function () {\r", - " pm.response.to.have.jsonBody(\"message\");\r", - "});" - ] - } - } - ] - } - ] - }, - { - "name": "aws-batch", - "item": [ - { - "name": "Positives", - "item": [ - { - "name": "job-submit", - "item": [ - { - "name": "logHelloWorld", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "var resp = pm.response.json()\r", - "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "logHelloWorld" - } - ] - } - }, - "response": [] - }, - { - "name": "job-dismiss", - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "logHelloWorld", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "var resp = pm.response.json()\r", - "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"]);\r", - "\r", - "setTimeout(() => {}, 40000); " - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "logHelloWorld" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"Response time is less than 1000ms\", function () {", - " pm.expect(pm.response.responseTime).to.be.below(1000);", - "});", - "", - "if (pm.response.responseTime > 500) {", - " console.log(`⚠️ Warning: Response time is ${pm.response.responseTime}ms which is greater than 500ms.`);", - "}" - ] - } - } - ] - }, - { - "name": "job-details", - "item": [ - { - "name": "mock-aws-batch-event-running", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - }, - { - "listen": "prerequest", - "script": { - "exec": [ - "postman.setGlobalVariable('timestampUtc', (new Date()).toISOString());" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"status\": \"running\",\n \"updated\": \"{{timestampUtc}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/jobs/:jobID/status", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "status" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-status", - "event": [ - { - "listen": "prerequest", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - }, - { - "listen": "test", - "script": { - "exec": [ - "pm.test(\"async process should be running or succeeded after 60 seconds\", function () {\r", - " pm.expect(pm.response.json()[\"status\"]).to.be.oneOf([\"running\",\"successful\"]);\r", - "});\r", - "\r", - "pm.test(\"response should have jobid, status, etc\", function () {\r", - " pm.response.to.not.be.error;\r", - " pm.response.to.have.jsonBody(\"jobID\");\r", - " pm.response.to.have.jsonBody(\"status\");\r", - " pm.response.to.have.jsonBody(\"updated\");\r", - " pm.response.to.have.jsonBody(\"processID\");\r", - "});\r", - "\r", - "setTimeout(() => {}, 50000); " - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "mock-aws-batch-event-successful", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - }, - { - "listen": "prerequest", - "script": { - "exec": [ - "postman.setGlobalVariable('timestampUtc', (new Date()).toISOString());" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"status\": \"successful\",\n \"updated\": \"{{timestampUtc}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/jobs/:jobID/status", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "status" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-logs", - "event": [ - { - "listen": "prerequest", - "script": { - "exec": [ - "setTimeout(() => {}, 30000); " - ], - "type": "text/javascript" - } - }, - { - "listen": "test", - "script": { - "exec": [ - "const logs = pm.response.json().process_logs;\r", - "\r", - "pm.test(\"number of process logs\", function () {\r", - " pm.expect(logs.length).to.eql(3001);\r", - "});\r", - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/logs", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "logs" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-results", - "event": [ - { - "listen": "prerequest", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/results", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "results" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-metadata", - "event": [ - { - "listen": "prerequest", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/metadata", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "metadata" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - } - ] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"Successful request\", function () {", - " pm.expect(pm.response.code).to.be.oneOf([200,201,202]);", - "});" - ] - } - } - ] - }, - { - "name": "job-submit", - "item": [ - { - "name": "logHelloWorld", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "var resp = pm.response.json()\r", - "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "logHelloWorld" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"Response time is less than 1000ms\", function () {", - " pm.expect(pm.response.responseTime).to.be.below(1000);", - "});", - "", - "if (pm.response.responseTime > 500) {", - " console.log(`⚠️ Warning: Response time is ${pm.response.responseTime}ms which is greater than 500ms.`);", - "}" - ] - } - } - ] - }, - { - "name": "Negatives", - "item": [ - { - "name": "job-logs-before-job-is-running", - "event": [ - { - "listen": "prerequest", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - }, - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/logs", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "logs" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-metadata", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/metadata", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "metadata" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-results", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/results", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "results" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"Status code is equal or greater than 400\", function () {", - " pm.expect(pm.response.code).to.be.greaterThan(399)", - "});", - "", - "pm.test(\"response should have message\", function () {", - " pm.response.to.have.jsonBody(\"message\");", - "});" - ] - } - } - ] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - } - ] - }, - { - "name": "execution-mode", - "item": [ - { - "name": "setup", - "item": [ - { - "name": "register-both-modes-process", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Process registered successfully', function () {", - " pm.expect(pm.response.code).to.be.oneOf([200, 201]);", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"info\": {\n \"version\": \"1.0.0\",\n \"id\": \"execModeBoth\",\n \"title\": \"Execution Mode Test (Both)\",\n \"description\": \"Supports both sync and async execution\",\n \"jobControlOptions\": [\"sync-execute\", \"async-execute\"],\n \"outputTransmission\": [\"value\"]\n },\n \"host\": {\n \"type\": \"subprocess\"\n },\n \"command\": [\"bash\", \"-c\", \"sleep 2 && echo '{\\\"plugin_results\\\": \\\"done\\\"}'\" ],\n \"config\": {\n \"maxResources\": {\n \"cpus\": 0.5,\n \"memory\": 256\n }\n },\n \"inputs\": [],\n \"outputs\": []\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/execModeBoth", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "execModeBoth" - ] - } - }, - "response": [] - }, - { - "name": "register-sync-only-process", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Process registered successfully', function () {", - " pm.expect(pm.response.code).to.be.oneOf([200, 201]);", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"info\": {\n \"version\": \"1.0.0\",\n \"id\": \"execModeSyncOnly\",\n \"title\": \"Execution Mode Test (Sync Only)\",\n \"description\": \"Supports only sync execution\",\n \"jobControlOptions\": [\"sync-execute\"],\n \"outputTransmission\": [\"value\"]\n },\n \"host\": {\n \"type\": \"subprocess\"\n },\n \"command\": [\"bash\", \"-c\", \"sleep 2 && echo '{\\\"plugin_results\\\": \\\"done\\\"}'\" ],\n \"config\": {\n \"maxResources\": {\n \"cpus\": 0.5,\n \"memory\": 256\n }\n },\n \"inputs\": [],\n \"outputs\": []\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/execModeSyncOnly", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "execModeSyncOnly" - ] - } - }, - "response": [] - }, - { - "name": "register-async-only-process", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Process registered successfully', function () {", - " pm.expect(pm.response.code).to.be.oneOf([200, 201]);", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"info\": {\n \"version\": \"1.0.0\",\n \"id\": \"execModeAsyncOnly\",\n \"title\": \"Execution Mode Test (Async Only)\",\n \"description\": \"Supports only async execution\",\n \"jobControlOptions\": [\"async-execute\"],\n \"outputTransmission\": [\"value\"]\n },\n \"host\": {\n \"type\": \"subprocess\"\n },\n \"command\": [\"bash\", \"-c\", \"sleep 2 && echo '{\\\"plugin_results\\\": \\\"done\\\"}'\" ],\n \"config\": {\n \"maxResources\": {\n \"cpus\": 0.5,\n \"memory\": 256\n }\n },\n \"inputs\": [],\n \"outputs\": []\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/execModeAsyncOnly", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "execModeAsyncOnly" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "req25-no-prefer-header", - "item": [ - { - "name": "Req25A-async-only-no-prefer", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "// Req 25A: async-only process without Prefer header -> async", - "pm.test('Async response (201) for async-only process', function () {", - " pm.response.to.have.status(201);", - "});", - "", - "pm.test('Job status is accepted', function () {", - " var resp = pm.response.json();", - " pm.expect(resp.status).to.eql('accepted');", - "});", - "", - "pm.test('No Preference-Applied header (no preference was requested)', function () {", - " pm.expect(pm.response.headers.has('Preference-Applied')).to.be.false;", - "});", - "", - "var resp = pm.response.json();", - "pm.collectionVariables.set('execModeJobID', resp.jobID);" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/execModeAsyncOnly/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "execModeAsyncOnly", - "execution" - ] - } - }, - "response": [] - }, - { - "name": "dismiss-job-25A", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Job dismissed', function () {", - " pm.response.to.have.status(200);", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/jobs/{{execModeJobID}}", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - "{{execModeJobID}}" - ] - } - }, - "response": [] - }, - { - "name": "Req25B-sync-only-no-prefer", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "// Req 25B: sync-only process without Prefer header -> sync", - "pm.test('Sync response (200) for sync-only process', function () {", - " pm.response.to.have.status(200);", - "});", - "", - "pm.test('Job completed successfully', function () {", - " var resp = pm.response.json();", - " pm.expect(resp.status).to.eql('successful');", - "});", - "", - "pm.test('No Preference-Applied header', function () {", - " pm.expect(pm.response.headers.has('Preference-Applied')).to.be.false;", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/execModeSyncOnly/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "execModeSyncOnly", - "execution" - ] - } - }, - "response": [] - }, - { - "name": "Req25C-both-modes-no-prefer", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "// Req 25C: both modes without Prefer header -> sync (default)", - "pm.test('Sync response (200) when both modes available and no preference', function () {", - " pm.response.to.have.status(200);", - "});", - "", - "pm.test('Job completed successfully', function () {", - " var resp = pm.response.json();", - " pm.expect(resp.status).to.eql('successful');", - "});", - "", - "pm.test('No Preference-Applied header', function () {", - " pm.expect(pm.response.headers.has('Preference-Applied')).to.be.false;", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/execModeBoth/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "execModeBoth", - "execution" - ] - } - }, - "response": [] - } - ], - "description": "OGC Requirement 25: Default execution mode when no Prefer header is present" - }, - { - "name": "req26-with-prefer-async", - "item": [ - { - "name": "Req26A-async-only-prefer-async", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "// Req 26A: async-only process with Prefer: respond-async -> async", - "pm.test('Async response (201) for async-only process', function () {", - " pm.response.to.have.status(201);", - "});", - "", - "pm.test('Job status is accepted', function () {", - " var resp = pm.response.json();", - " pm.expect(resp.status).to.eql('accepted');", - "});", - "", - "pm.test('No Preference-Applied (process only supports async anyway)', function () {", - " pm.expect(pm.response.headers.has('Preference-Applied')).to.be.false;", - "});", - "", - "var resp = pm.response.json();", - "pm.collectionVariables.set('execModeJobID', resp.jobID);" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Prefer", - "value": "respond-async" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/execModeAsyncOnly/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "execModeAsyncOnly", - "execution" - ] - } - }, - "response": [] - }, - { - "name": "dismiss-job-26A", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Job dismissed', function () {", - " pm.response.to.have.status(200);", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/jobs/{{execModeJobID}}", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - "{{execModeJobID}}" - ] - } - }, - "response": [] - }, - { - "name": "Req26B-sync-only-prefer-async", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "// Req 26B: sync-only process with Prefer: respond-async -> sync (ignore preference)", - "pm.test('Sync response (200) - preference ignored for sync-only process', function () {", - " pm.response.to.have.status(200);", - "});", - "", - "pm.test('Job completed successfully', function () {", - " var resp = pm.response.json();", - " pm.expect(resp.status).to.eql('successful');", - "});", - "", - "pm.test('No Preference-Applied header (preference was not honored)', function () {", - " pm.expect(pm.response.headers.has('Preference-Applied')).to.be.false;", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Prefer", - "value": "respond-async" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/execModeSyncOnly/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "execModeSyncOnly", - "execution" - ] - } - }, - "response": [] - }, - { - "name": "Req26C-Rec12A-both-modes-prefer-async", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "// Req 26C + Rec 12A: both modes with Prefer: respond-async -> async (honor preference)", - "pm.test('Async response (201) when preference is honored', function () {", - " pm.response.to.have.status(201);", - "});", - "", - "pm.test('Job status is accepted', function () {", - " var resp = pm.response.json();", - " pm.expect(resp.status).to.eql('accepted');", - "});", - "", - "pm.test('Preference-Applied header is set (Rec 14)', function () {", - " pm.expect(pm.response.headers.has('Preference-Applied')).to.be.true;", - " pm.expect(pm.response.headers.get('Preference-Applied')).to.eql('respond-async');", - "});", - "", - "var resp = pm.response.json();", - "pm.collectionVariables.set('execModeJobID', resp.jobID);" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Prefer", - "value": "respond-async" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/execModeBoth/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "execModeBoth", - "execution" - ] - } - }, - "response": [] - }, - { - "name": "dismiss-job-26C", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Job dismissed', function () {", - " pm.response.to.have.status(200);", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/jobs/{{execModeJobID}}", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - "{{execModeJobID}}" - ] - } - }, - "response": [] - } - ], - "description": "OGC Requirement 26: Execution mode when Prefer: respond-async header is present" - }, - { - "name": "cleanup", - "item": [ - { - "name": "delete-both-modes-process", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Process deleted successfully', function () {", - " pm.response.to.have.status(200);", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/processes/execModeBoth", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "execModeBoth" - ] - } - }, - "response": [] - }, - { - "name": "delete-sync-only-process", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Process deleted successfully', function () {", - " pm.response.to.have.status(200);", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/processes/execModeSyncOnly", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "execModeSyncOnly" - ] - } - }, - "response": [] - }, - { - "name": "delete-async-only-process", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Process deleted successfully', function () {", - " pm.response.to.have.status(200);", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/processes/execModeAsyncOnly", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "execModeAsyncOnly" - ] - } - }, - "response": [] - } - ] - } - ], - "description": "Tests for OGC API - Processes execution mode selection (Requirements 25/26, Recommendation 12A/14)" - } - ] - }, - { - "name": "jobs-list", - "item": [ - { - "name": "Positives", - "item": [ - { - "name": "number-of-jobs", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "const jobsData = pm.response.json().jobs;", - "", - "pm.test(\"number of jobs in record test\", function () {", - " pm.expect(jobsData.length).to.eql(11);", - " });" - ], - "type": "text/javascript", - "packages": {} - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs?", - "host": [ - "{{url}}" - ], - "path": [ - "jobs" - ], - "query": [ - { - "key": "", - "value": null - } - ] - } - }, - "response": [] - }, - { - "name": "fetch-with-status", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "const jobsData = pm.response.json().jobs;", - "pm.test(\"All jobs have the 'successful' status\", function () {", - " jobsData.forEach(job => {", - " pm.expect(job.status).to.eql('successful');", - " });", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs?status=successful", - "host": [ - "{{url}}" - ], - "path": [ - "jobs" - ], - "query": [ - { - "key": "status", - "value": "successful" - } - ] - } - }, - "response": [] - }, - { - "name": "fetch-with-processID", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "const jobsData = pm.response.json().jobs;", - "pm.test(\"All jobs have the 'dfc' processID\", function () {", - " jobsData.forEach(job => {", - " pm.expect(job.processID).to.eql('dfc');", - " });", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs?processID=dfc", - "host": [ - "{{url}}" - ], - "path": [ - "jobs" - ], - "query": [ - { - "key": "processID", - "value": "dfc" - } - ] - } - }, - "response": [] - }, - { - "name": "fetch-with-multiple-statuses", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs?status=successful,failed", - "host": [ - "{{url}}" - ], - "path": [ - "jobs" - ], - "query": [ - { - "key": "status", - "value": "successful,failed" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test('Status code is 200 ', function(){", - " pm.response.to.have.status(200)", - "});" - ] - } - } - ] - }, - { - "name": "Negatives", - "item": [ - { - "name": "fetch-with-invalid-status", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs?status=invalidStatus", - "host": [ - "{{url}}" - ], - "path": [ - "jobs" - ], - "query": [ - { - "key": "status", - "value": "invalidStatus" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"Status code is equal or greater than 400\", function () {", - " pm.expect(pm.response.code).to.be.greaterThan(399)", - "});", - "", - "pm.test(\"response should have message\", function () {", - " pm.response.to.have.jsonBody(\"message\");", - "});" - ] - } - } - ] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "pm.test(\"Response time is less than 1000ms\", function () {", - " pm.expect(pm.response.responseTime).to.be.below(1000);", - "});", - "", - "if (pm.response.responseTime > 500) {", - " console.log(`⚠️ Warning: Response time is ${pm.response.responseTime}ms which is greater than 500ms.`);", - "}" - ] - } - } - ] - }, - { - "name": "scheduler", - "item": [ - { - "name": "setup", - "item": [ - { - "name": "register-sleeper-process", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Process registered successfully', function () {", - " pm.expect(pm.response.code).to.be.oneOf([200, 201]);", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"info\": {\n \"version\": \"1.0.0\",\n \"id\": \"sleeperScheduler\",\n \"title\": \"Sleeper Scheduler Test\",\n \"description\": \"A test process for scheduler resource contention (supports both sync and async)\",\n \"jobControlOptions\": [\"sync-execute\", \"async-execute\"],\n \"outputTransmission\": [\"value\"]\n },\n \"host\": {\n \"type\": \"subprocess\"\n },\n \"command\": [\"bash\", \"-c\", \"sleep 10 && echo '{\\\"plugin_results\\\": \\\"slept for 10 seconds!\\\"}'\" ],\n \"config\": {\n \"maxResources\": {\n \"cpus\": 4.0,\n \"memory\": 4096\n }\n },\n \"inputs\": [],\n \"outputs\": []\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/sleeperScheduler", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "sleeperScheduler" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "resource-contention", - "item": [ - { - "name": "submit-async-sleeper", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Status code is 201', function () {", - " pm.response.to.have.status(201);", - "});", - "", - "pm.test('Preference-Applied header is set', function () {", - " pm.expect(pm.response.headers.get('Preference-Applied')).to.eql('respond-async');", - "});", - "", - "var resp = pm.response.json();", - "pm.collectionVariables.set('sleeperJobID', resp.jobID);" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [ - { - "key": "Prefer", - "value": "respond-async" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/sleeperScheduler/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "sleeperScheduler", - "execution" - ] - } - }, - "response": [] - }, - { - "name": "check-resources-in-use", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Status code is 200', function () {", - " pm.response.to.have.status(200);", - "});", - "", - "pm.test('Resources are in use or queued', function () {", - " var resources = pm.response.json().resources;", - " var totalCPUs = resources.usedCPUs + resources.queuedCPUs;", - " pm.expect(totalCPUs).to.be.above(0);", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/admin/resources", - "host": [ - "{{url}}" - ], - "path": [ - "admin", - "resources" - ] - } - }, - "response": [] - }, - { - "name": "sync-job-fails-when-resources-full", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Sync job returns 503 when resources unavailable', function () {", - " pm.response.to.have.status(503);", - "});", - "", - "pm.test('Response has appropriate error message', function () {", - " var resp = pm.response.json();", - " pm.expect(resp.message).to.include('resources');", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/sleeperScheduler/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "sleeperScheduler", - "execution" - ] - } - }, - "response": [] - }, - { - "name": "sync-job-succeeds-after-wait", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Sync job succeeds when resources available', function () {", - " pm.response.to.have.status(200);", - "});", - "", - "pm.test('Job completed successfully', function () {", - " var resp = pm.response.json();", - " pm.expect(resp.status).to.eql('successful');", - "});" - ], - "type": "text/javascript", - "packages": {}, - "requests": {} - } - }, - { - "listen": "prerequest", - "script": { - "exec": [ - "// Wait 10 seconds for the async sleeper job to complete", - "setTimeout(() => {}, 10000);" - ], - "type": "text/javascript", - "packages": {}, - "requests": {} - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/sleeperScheduler/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "sleeperScheduler", - "execution" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "cleanup", - "item": [ - { - "name": "delete-sleeper-scheduler", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test('Process deleted successfully', function () {", - " pm.response.to.have.status(200);", - "});" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/processes/sleeperScheduler", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "sleeperScheduler" - ] - } - }, - "response": [] - } - ] - } - ], - "description": "Tests for the local scheduler resource management" - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "requests": {}, - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "requests": {}, - "exec": [ - "pm.test(\"response must be valid and have a body\", function () {", - " pm.response.to.be.withBody;", - " pm.response.to.be.json;", - "});" - ] - } - } - ], - "variable": [ - { - "key": "url", - "value": "localhost:5050" - }, - { - "key": "jobID", - "value": "" - }, - { - "key": "sleeperJobID", - "value": "" - }, - { - "key": "execModeJobID", - "value": "" - }, - { - "key": "maxCPUs", - "value": "" - }, - { - "key": "maxMemory", - "value": "" - } - ] -} \ No newline at end of file + "info": { + "_postman_id": "e4cfedee-5b49-464a-beba-6a8a9648e76e", + "name": "sepex Testing", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "36773789" + }, + "item": [ + { + "name": "admin", + "item": [ + { + "name": "resources", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Status code is 200', function () {", + " pm.response.to.have.status(200);", + "});", + "", + "pm.test('Response has resources object', function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData).to.have.property('resources');", + "});", + "", + "pm.test('Response has links array', function () {", + " var jsonData = pm.response.json();", + " pm.expect(jsonData).to.have.property('links');", + " pm.expect(jsonData.links).to.be.an('array');", + "});", + "", + "pm.test('Resources has required fields', function () {", + " var resources = pm.response.json().resources;", + " pm.expect(resources).to.have.property('usedCPUs');", + " pm.expect(resources).to.have.property('usedMemory');", + " pm.expect(resources).to.have.property('queuedCPUs');", + " pm.expect(resources).to.have.property('queuedMemory');", + " pm.expect(resources).to.have.property('maxCPUs');", + " pm.expect(resources).to.have.property('maxMemory');", + " pm.expect(resources).to.have.property('usedCPUsPct');", + " pm.expect(resources).to.have.property('queuedCPUsPct');", + " pm.expect(resources).to.have.property('usedMemPct');", + " pm.expect(resources).to.have.property('queuedMemPct');", + "});", + "", + "pm.test('Percentage values are valid numbers', function () {", + " var resources = pm.response.json().resources;", + " pm.expect(resources.usedCPUsPct).to.be.a('number');", + " pm.expect(resources.usedMemPct).to.be.a('number');", + " pm.expect(resources.queuedCPUsPct).to.be.a('number');", + " pm.expect(resources.queuedMemPct).to.be.a('number');", + "});", + "", + "pm.test('Max values are greater than 0', function () {", + " var resources = pm.response.json().resources;", + " pm.expect(resources.maxCPUs).to.be.above(0);", + " pm.expect(resources.maxMemory).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/admin/resources", + "host": [ + "{{url}}" + ], + "path": [ + "admin", + "resources" + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Response time is less than 1000ms\", function () {", + " pm.expect(pm.response.responseTime).to.be.below(1000);", + "});" + ] + } + } + ] + }, + { + "name": "startup", + "item": [ + { + "name": "landingpage", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/", + "host": [ + "{{url}}" + ], + "path": [ + "" + ] + } + }, + "response": [] + }, + { + "name": "conformance", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/conformance", + "host": [ + "{{url}}" + ], + "path": [ + "conformance" + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Status code is 200 ', function(){", + " pm.response.to.have.status(200)", + "});", + "", + "pm.test(\"Response time is less than 1000ms\", function () {", + " pm.expect(pm.response.responseTime).to.be.below(1000);", + "});", + "", + "if (pm.response.responseTime > 100) {", + " console.log(`⚠️ Warning: Response time is ${pm.response.responseTime}ms which is greater than 100ms.`);", + "}" + ] + } + } + ] + }, + { + "name": "processes", + "item": [ + { + "name": "processes", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "const processData = pm.response.json().processes;\r", + "\r", + "pm.test(\"number of processes registered test\", function () {\r", + " pm.expect(processData.length).to.eql(1);\r", + "});\r", + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/processes?limit=1&offset=1", + "host": [ + "{{url}}" + ], + "path": [ + "processes" + ], + "query": [ + { + "key": "limit", + "value": "1" + }, + { + "key": "offset", + "value": "1" + } + ] + } + }, + "response": [] + }, + { + "name": "process-describe", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "const processData = pm.response.json();\r", + "\r", + "pm.test(\"id test\", function () {\r", + " pm.expect(processData[\"info\"][\"id\"]).to.eql(\"pyecho\");\r", + "});\r", + "\r", + "pm.test(\"job control test\", function () {\r", + " pm.expect(processData[\"info\"][\"jobControlOptions\"][0]).to.eql(\"sync-execute\");\r", + "});" + ], + "type": "text/javascript", + "packages": {} + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/processes/:processID", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID" + ], + "variable": [ + { + "key": "processID", + "value": "pyecho" + } + ] + } + }, + "response": [] + }, + { + "name": "delete-startup-process", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/processes/:processID", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID" + ], + "variable": [ + { + "key": "processID", + "value": "pywrite" + } + ] + } + }, + "response": [] + }, + { + "name": "add-process", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\r\n \"info\": {\r\n \"version\": \"12.5.2023\",\r\n \"id\": \"dfcTest\",\r\n \"title\": \"Depth Frequency Curve\",\r\n \"description\": \"Returns data for depth-frequency curve\",\r\n \"jobControlOptions\": [\"async-execute\"],\r\n \"outputTransmission\": [\"value\"]\r\n },\r\n \"host\": {\r\n \"type\": \"aws-batch\",\r\n \"jobDefinition\": \"process-sandbox:2\",\r\n \"jobQueue\": \"micro-test\"\r\n },\r\n \"command\": [\"python\", \"dfc.py\"],\r\n \"inputs\": [\r\n {\r\n \"id\": \"crs\",\r\n \"title\": \"crs\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 0,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"tile\",\r\n \"title\": \"tile\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"epoch\",\r\n \"title\": \"epoch\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"points\",\r\n \"title\": \"points\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 999999\r\n }\r\n ],\r\n \"outputs\": [\r\n {\r\n \"id\": \"dfc\",\r\n \"title\": \"dfc\",\r\n \"description\": \"\",\r\n \"output\": {\r\n \"transmissionMode\": [\r\n \"value\"\r\n ]\r\n }\r\n }\r\n ]\r\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID" + ], + "variable": [ + { + "key": "processID", + "value": "dfcTest" + } + ] + } + }, + "response": [] + }, + { + "name": "update-process", + "request": { + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\r\n \"info\": {\r\n \"version\": \"12.5.2023\",\r\n \"id\": \"dfcTest\",\r\n \"title\": \"Updated Depth Frequency Curve\",\r\n \"description\": \"Returns data for depth-frequency curve\",\r\n \"jobControlOptions\": [\"async-execute\"],\r\n \"outputTransmission\": [\"value\"]\r\n },\r\n \"host\": {\r\n \"type\": \"aws-batch\",\r\n \"jobDefinition\": \"process-sandbox:2\",\r\n \"jobQueue\": \"micro-test\"\r\n },\r\n \"command\": [\"python\", \"dfc.py\"],\r\n \"inputs\": [\r\n {\r\n \"id\": \"crs\",\r\n \"title\": \"crs\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 0,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"tile\",\r\n \"title\": \"tile\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"epoch\",\r\n \"title\": \"epoch\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"points\",\r\n \"title\": \"points\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 999999\r\n }\r\n ],\r\n \"outputs\": [\r\n {\r\n \"id\": \"dfc\",\r\n \"title\": \"dfc\",\r\n \"description\": \"\",\r\n \"output\": {\r\n \"transmissionMode\": [\r\n \"value\"\r\n ]\r\n }\r\n }\r\n ]\r\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID" + ], + "variable": [ + { + "key": "processID", + "value": "dfcTest" + } + ] + } + }, + "response": [] + }, + { + "name": "delete-process", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/processes/:processID", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID" + ], + "variable": [ + { + "key": "processID", + "value": "dfcTest" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Status code is 200 ', function(){", + " pm.response.to.have.status(200)", + "});", + "", + "pm.test(\"Response time is less than 1000ms\", function () {", + " pm.expect(pm.response.responseTime).to.be.below(1000);", + "});", + "", + "if (pm.response.responseTime > 100) {", + " console.log(`⚠️ Warning: Response time is ${pm.response.responseTime}ms which is greater than 100ms.`);", + "}" + ] + } + } + ] + }, + { + "name": "execution", + "item": [ + { + "name": "general", + "item": [ + { + "name": "Negatives", + "item": [ + { + "name": "missing", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {\n \"crs\": \"4326\",\n \"points\": [\n [\n -76.491824,\n 37.271065\n ]\n ],\n \"epoch\": \"2020\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "dfc" + } + ] + } + }, + "response": [] + }, + { + "name": "incorrectKey", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {\n \"incorrectKey\": \"value\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "pyecho" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status code is equal or greater than 400\", function () {", + " pm.expect(pm.response.code).to.be.greaterThan(399)", + "});", + "", + "pm.test(\"response should have message\", function () {", + " pm.response.to.have.jsonBody(\"message\");", + "});" + ] + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ] + }, + { + "name": "sync-docker", + "item": [ + { + "name": "Positives", + "item": [ + { + "name": "job-submit", + "item": [ + { + "name": "pyecho", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {\n \"text\": \"Hello World!\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "pyecho" + } + ] + } + }, + "response": [] + }, + { + "name": "pyecho-with-tags", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var resp = pm.response.json();", + "pm.collectionVariables.set(\"taggedJobID\", resp[\"jobID\"]);", + "pm.test(\"job submit with tags returns jobID\", function () {", + " pm.response.to.have.status(200);", + " pm.expect(resp).to.have.property(\"jobID\");", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {\n \"text\": \"Hello World!\"\n },\n \"tags\": [\"texas\", \"huc984727\"]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "pyecho" + } + ] + } + }, + "response": [] + }, + { + "name": "pyecho-with-empty-tags", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var resp = pm.response.json();", + "pm.collectionVariables.set(\"emptyTagsJobID\", resp[\"jobID\"]);", + "pm.test(\"job submit with empty tags returns jobID\", function () {", + " pm.response.to.have.status(200);", + " pm.expect(resp).to.have.property(\"jobID\");", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {\n \"text\": \"Hello World!\"\n },\n \"tags\": []\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "pyecho" + } + ] + } + }, + "response": [] + }, + { + "name": "pyecho-without-tags", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var resp = pm.response.json();", + "pm.collectionVariables.set(\"noTagsJobID\", resp[\"jobID\"]);", + "pm.test(\"job submit without tags returns jobID\", function () {", + " pm.response.to.have.status(200);", + " pm.expect(resp).to.have.property(\"jobID\");", + "});" + ] + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {\n \"text\": \"Hello World!\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "pyecho" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var resp = pm.response.json()", + "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" + ] + } + } + ] + }, + { + "name": "job-details", + "item": [ + { + "name": "job-status", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"response should have jobid, status, etc\", function () {\r", + " pm.response.to.not.be.error;\r", + " pm.response.to.have.jsonBody(\"jobID\");\r", + " pm.response.to.have.jsonBody(\"status\");\r", + " pm.response.to.have.jsonBody(\"updated\");\r", + " pm.response.to.have.jsonBody(\"processID\");\r", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-logs", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/logs", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "logs" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-results", + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/results", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "results" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-metadata", + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/metadata", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "metadata" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Status code is 200', function(){", + " pm.response.to.have.status(200)", + "});", + "", + "" + ] + } + } + ] + }, + { + "name": "Negatives", + "item": [ + { + "name": "job-dismiss", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status code is equal or greater than 400\", function () {", + " pm.expect(pm.response.code).to.be.greaterThan(399)", + "});", + "", + "pm.test(\"response should have message\", function () {", + " pm.response.to.have.jsonBody(\"message\");", + "});" + ] + } + } + ] + } + ] + }, + { + "name": "sync-subprocess", + "item": [ + { + "name": "Positives", + "item": [ + { + "name": "job-submit", + "item": [ + { + "name": "subprocessEcho", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript", + "packages": {} + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "subprocessEcho" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "packages": {}, + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "packages": {}, + "exec": [ + "var resp = pm.response.json()\r", + "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" + ] + } + } + ] + }, + { + "name": "job-details", + "item": [ + { + "name": "job-status", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"response should have jobid, status, etc\", function () {\r", + " pm.response.to.not.be.error;\r", + " pm.response.to.have.jsonBody(\"jobID\");\r", + " pm.response.to.have.jsonBody(\"status\");\r", + " pm.response.to.have.jsonBody(\"updated\");\r", + " pm.response.to.have.jsonBody(\"processID\");\r", + "});" + ], + "type": "text/javascript", + "packages": {} + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-logs", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/logs", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "logs" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-results", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/results", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "results" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-metadata", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/metadata", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "metadata" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "packages": {}, + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "packages": {}, + "exec": [ + "pm.test('Status code is 200', function(){\r", + " pm.response.to.have.status(200)\r", + "});\r", + "\r", + "" + ] + } + } + ] + }, + { + "name": "Negatives", + "item": [ + { + "name": "job-dismiss", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "packages": {}, + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "packages": {}, + "exec": [ + "pm.test(\"Status code is equal or greater than 400\", function () {\r", + " pm.expect(pm.response.code).to.be.greaterThan(399)\r", + "});\r", + "\r", + "pm.test(\"response should have message\", function () {\r", + " pm.response.to.have.jsonBody(\"message\");\r", + "});" + ] + } + } + ] + } + ] + }, + { + "name": "aws-batch", + "item": [ + { + "name": "Positives", + "item": [ + { + "name": "job-submit", + "item": [ + { + "name": "logHelloWorld", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "var resp = pm.response.json()\r", + "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "logHelloWorld" + } + ] + } + }, + "response": [] + }, + { + "name": "job-dismiss", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "logHelloWorld", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "var resp = pm.response.json()\r", + "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"]);\r", + "\r", + "setTimeout(() => {}, 40000); " + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "logHelloWorld" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Response time is less than 1000ms\", function () {", + " pm.expect(pm.response.responseTime).to.be.below(1000);", + "});", + "", + "if (pm.response.responseTime > 500) {", + " console.log(`⚠️ Warning: Response time is ${pm.response.responseTime}ms which is greater than 500ms.`);", + "}" + ] + } + } + ] + }, + { + "name": "job-details", + "item": [ + { + "name": "mock-aws-batch-event-running", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + }, + { + "listen": "prerequest", + "script": { + "exec": [ + "postman.setGlobalVariable('timestampUtc', (new Date()).toISOString());" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"status\": \"running\",\n \"updated\": \"{{timestampUtc}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/jobs/:jobID/status", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "status" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-status", + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + }, + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"async process should be running or succeeded after 60 seconds\", function () {\r", + " pm.expect(pm.response.json()[\"status\"]).to.be.oneOf([\"running\",\"successful\"]);\r", + "});\r", + "\r", + "pm.test(\"response should have jobid, status, etc\", function () {\r", + " pm.response.to.not.be.error;\r", + " pm.response.to.have.jsonBody(\"jobID\");\r", + " pm.response.to.have.jsonBody(\"status\");\r", + " pm.response.to.have.jsonBody(\"updated\");\r", + " pm.response.to.have.jsonBody(\"processID\");\r", + "});\r", + "\r", + "setTimeout(() => {}, 50000); " + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "mock-aws-batch-event-successful", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + }, + { + "listen": "prerequest", + "script": { + "exec": [ + "postman.setGlobalVariable('timestampUtc', (new Date()).toISOString());" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"status\": \"successful\",\n \"updated\": \"{{timestampUtc}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/jobs/:jobID/status", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "status" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-logs", + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + "setTimeout(() => {}, 30000); " + ], + "type": "text/javascript" + } + }, + { + "listen": "test", + "script": { + "exec": [ + "const logs = pm.response.json().process_logs;\r", + "\r", + "pm.test(\"number of process logs\", function () {\r", + " pm.expect(logs.length).to.eql(3001);\r", + "});\r", + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/logs", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "logs" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-results", + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/results", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "results" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-metadata", + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/metadata", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "metadata" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Successful request\", function () {", + " pm.expect(pm.response.code).to.be.oneOf([200,201,202]);", + "});" + ] + } + } + ] + }, + { + "name": "job-submit", + "item": [ + { + "name": "logHelloWorld", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "var resp = pm.response.json()\r", + "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "logHelloWorld" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Response time is less than 1000ms\", function () {", + " pm.expect(pm.response.responseTime).to.be.below(1000);", + "});", + "", + "if (pm.response.responseTime > 500) {", + " console.log(`⚠️ Warning: Response time is ${pm.response.responseTime}ms which is greater than 500ms.`);", + "}" + ] + } + } + ] + }, + { + "name": "Negatives", + "item": [ + { + "name": "job-logs-before-job-is-running", + "event": [ + { + "listen": "prerequest", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + }, + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/logs", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "logs" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-metadata", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/metadata", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "metadata" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-results", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/results", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "results" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status code is equal or greater than 400\", function () {", + " pm.expect(pm.response.code).to.be.greaterThan(399)", + "});", + "", + "pm.test(\"response should have message\", function () {", + " pm.response.to.have.jsonBody(\"message\");", + "});" + ] + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ] + }, + { + "name": "execution-mode", + "item": [ + { + "name": "setup", + "item": [ + { + "name": "register-both-modes-process", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Process registered successfully', function () {", + " pm.expect(pm.response.code).to.be.oneOf([200, 201]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"info\": {\n \"version\": \"1.0.0\",\n \"id\": \"execModeBoth\",\n \"title\": \"Execution Mode Test (Both)\",\n \"description\": \"Supports both sync and async execution\",\n \"jobControlOptions\": [\"sync-execute\", \"async-execute\"],\n \"outputTransmission\": [\"value\"]\n },\n \"host\": {\n \"type\": \"subprocess\"\n },\n \"command\": [\"bash\", \"-c\", \"sleep 2 && echo '{\\\"plugin_results\\\": \\\"done\\\"}'\" ],\n \"config\": {\n \"maxResources\": {\n \"cpus\": 0.5,\n \"memory\": 256\n }\n },\n \"inputs\": [],\n \"outputs\": []\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/execModeBoth", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "execModeBoth" + ] + } + }, + "response": [] + }, + { + "name": "register-sync-only-process", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Process registered successfully', function () {", + " pm.expect(pm.response.code).to.be.oneOf([200, 201]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"info\": {\n \"version\": \"1.0.0\",\n \"id\": \"execModeSyncOnly\",\n \"title\": \"Execution Mode Test (Sync Only)\",\n \"description\": \"Supports only sync execution\",\n \"jobControlOptions\": [\"sync-execute\"],\n \"outputTransmission\": [\"value\"]\n },\n \"host\": {\n \"type\": \"subprocess\"\n },\n \"command\": [\"bash\", \"-c\", \"sleep 2 && echo '{\\\"plugin_results\\\": \\\"done\\\"}'\" ],\n \"config\": {\n \"maxResources\": {\n \"cpus\": 0.5,\n \"memory\": 256\n }\n },\n \"inputs\": [],\n \"outputs\": []\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/execModeSyncOnly", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "execModeSyncOnly" + ] + } + }, + "response": [] + }, + { + "name": "register-async-only-process", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Process registered successfully', function () {", + " pm.expect(pm.response.code).to.be.oneOf([200, 201]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"info\": {\n \"version\": \"1.0.0\",\n \"id\": \"execModeAsyncOnly\",\n \"title\": \"Execution Mode Test (Async Only)\",\n \"description\": \"Supports only async execution\",\n \"jobControlOptions\": [\"async-execute\"],\n \"outputTransmission\": [\"value\"]\n },\n \"host\": {\n \"type\": \"subprocess\"\n },\n \"command\": [\"bash\", \"-c\", \"sleep 2 && echo '{\\\"plugin_results\\\": \\\"done\\\"}'\" ],\n \"config\": {\n \"maxResources\": {\n \"cpus\": 0.5,\n \"memory\": 256\n }\n },\n \"inputs\": [],\n \"outputs\": []\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/execModeAsyncOnly", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "execModeAsyncOnly" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "req25-no-prefer-header", + "item": [ + { + "name": "Req25A-async-only-no-prefer", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "// Req 25A: async-only process without Prefer header -> async", + "pm.test('Async response (201) for async-only process', function () {", + " pm.response.to.have.status(201);", + "});", + "", + "pm.test('Job status is accepted', function () {", + " var resp = pm.response.json();", + " pm.expect(resp.status).to.eql('accepted');", + "});", + "", + "pm.test('No Preference-Applied header (no preference was requested)', function () {", + " pm.expect(pm.response.headers.has('Preference-Applied')).to.be.false;", + "});", + "", + "var resp = pm.response.json();", + "pm.collectionVariables.set('execModeJobID', resp.jobID);" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/execModeAsyncOnly/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "execModeAsyncOnly", + "execution" + ] + } + }, + "response": [] + }, + { + "name": "dismiss-job-25A", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Job dismissed', function () {", + " pm.response.to.have.status(200);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/jobs/{{execModeJobID}}", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + "{{execModeJobID}}" + ] + } + }, + "response": [] + }, + { + "name": "Req25B-sync-only-no-prefer", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "// Req 25B: sync-only process without Prefer header -> sync", + "pm.test('Sync response (200) for sync-only process', function () {", + " pm.response.to.have.status(200);", + "});", + "", + "pm.test('Job completed successfully', function () {", + " var resp = pm.response.json();", + " pm.expect(resp.status).to.eql('successful');", + "});", + "", + "pm.test('No Preference-Applied header', function () {", + " pm.expect(pm.response.headers.has('Preference-Applied')).to.be.false;", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/execModeSyncOnly/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "execModeSyncOnly", + "execution" + ] + } + }, + "response": [] + }, + { + "name": "Req25C-both-modes-no-prefer", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "// Req 25C: both modes without Prefer header -> sync (default)", + "pm.test('Sync response (200) when both modes available and no preference', function () {", + " pm.response.to.have.status(200);", + "});", + "", + "pm.test('Job completed successfully', function () {", + " var resp = pm.response.json();", + " pm.expect(resp.status).to.eql('successful');", + "});", + "", + "pm.test('No Preference-Applied header', function () {", + " pm.expect(pm.response.headers.has('Preference-Applied')).to.be.false;", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/execModeBoth/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "execModeBoth", + "execution" + ] + } + }, + "response": [] + } + ], + "description": "OGC Requirement 25: Default execution mode when no Prefer header is present" + }, + { + "name": "req26-with-prefer-async", + "item": [ + { + "name": "Req26A-async-only-prefer-async", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "// Req 26A: async-only process with Prefer: respond-async -> async", + "pm.test('Async response (201) for async-only process', function () {", + " pm.response.to.have.status(201);", + "});", + "", + "pm.test('Job status is accepted', function () {", + " var resp = pm.response.json();", + " pm.expect(resp.status).to.eql('accepted');", + "});", + "", + "pm.test('No Preference-Applied (process only supports async anyway)', function () {", + " pm.expect(pm.response.headers.has('Preference-Applied')).to.be.false;", + "});", + "", + "var resp = pm.response.json();", + "pm.collectionVariables.set('execModeJobID', resp.jobID);" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Prefer", + "value": "respond-async" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/execModeAsyncOnly/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "execModeAsyncOnly", + "execution" + ] + } + }, + "response": [] + }, + { + "name": "dismiss-job-26A", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Job dismissed', function () {", + " pm.response.to.have.status(200);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/jobs/{{execModeJobID}}", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + "{{execModeJobID}}" + ] + } + }, + "response": [] + }, + { + "name": "Req26B-sync-only-prefer-async", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "// Req 26B: sync-only process with Prefer: respond-async -> sync (ignore preference)", + "pm.test('Sync response (200) - preference ignored for sync-only process', function () {", + " pm.response.to.have.status(200);", + "});", + "", + "pm.test('Job completed successfully', function () {", + " var resp = pm.response.json();", + " pm.expect(resp.status).to.eql('successful');", + "});", + "", + "pm.test('No Preference-Applied header (preference was not honored)', function () {", + " pm.expect(pm.response.headers.has('Preference-Applied')).to.be.false;", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Prefer", + "value": "respond-async" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/execModeSyncOnly/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "execModeSyncOnly", + "execution" + ] + } + }, + "response": [] + }, + { + "name": "Req26C-Rec12A-both-modes-prefer-async", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "// Req 26C + Rec 12A: both modes with Prefer: respond-async -> async (honor preference)", + "pm.test('Async response (201) when preference is honored', function () {", + " pm.response.to.have.status(201);", + "});", + "", + "pm.test('Job status is accepted', function () {", + " var resp = pm.response.json();", + " pm.expect(resp.status).to.eql('accepted');", + "});", + "", + "pm.test('Preference-Applied header is set (Rec 14)', function () {", + " pm.expect(pm.response.headers.has('Preference-Applied')).to.be.true;", + " pm.expect(pm.response.headers.get('Preference-Applied')).to.eql('respond-async');", + "});", + "", + "var resp = pm.response.json();", + "pm.collectionVariables.set('execModeJobID', resp.jobID);" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Prefer", + "value": "respond-async" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/execModeBoth/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "execModeBoth", + "execution" + ] + } + }, + "response": [] + }, + { + "name": "dismiss-job-26C", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Job dismissed', function () {", + " pm.response.to.have.status(200);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/jobs/{{execModeJobID}}", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + "{{execModeJobID}}" + ] + } + }, + "response": [] + } + ], + "description": "OGC Requirement 26: Execution mode when Prefer: respond-async header is present" + }, + { + "name": "cleanup", + "item": [ + { + "name": "delete-both-modes-process", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Process deleted successfully', function () {", + " pm.response.to.have.status(200);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/processes/execModeBoth", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "execModeBoth" + ] + } + }, + "response": [] + }, + { + "name": "delete-sync-only-process", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Process deleted successfully', function () {", + " pm.response.to.have.status(200);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/processes/execModeSyncOnly", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "execModeSyncOnly" + ] + } + }, + "response": [] + }, + { + "name": "delete-async-only-process", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Process deleted successfully', function () {", + " pm.response.to.have.status(200);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/processes/execModeAsyncOnly", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "execModeAsyncOnly" + ] + } + }, + "response": [] + } + ] + } + ], + "description": "Tests for OGC API - Processes execution mode selection (Requirements 25/26, Recommendation 12A/14)" + } + ] + }, + { + "name": "jobs-list", + "item": [ + { + "name": "Positives", + "item": [ + { + "name": "number-of-jobs", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "const jobsData = pm.response.json().jobs;", + "", + "pm.test(\"number of jobs in record test\", function () {", + " pm.expect(jobsData.length).to.eql(11);", + " });" + ], + "type": "text/javascript", + "packages": {} + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs?", + "host": [ + "{{url}}" + ], + "path": [ + "jobs" + ], + "query": [ + { + "key": "", + "value": null + } + ] + } + }, + "response": [] + }, + { + "name": "fetch-with-status", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "const jobsData = pm.response.json().jobs;", + "pm.test(\"All jobs have the 'successful' status\", function () {", + " jobsData.forEach(job => {", + " pm.expect(job.status).to.eql('successful');", + " });", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs?status=successful", + "host": [ + "{{url}}" + ], + "path": [ + "jobs" + ], + "query": [ + { + "key": "status", + "value": "successful" + } + ] + } + }, + "response": [] + }, + { + "name": "fetch-with-processID", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "const jobsData = pm.response.json().jobs;", + "pm.test(\"All jobs have the 'dfc' processID\", function () {", + " jobsData.forEach(job => {", + " pm.expect(job.processID).to.eql('dfc');", + " });", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs?processID=dfc", + "host": [ + "{{url}}" + ], + "path": [ + "jobs" + ], + "query": [ + { + "key": "processID", + "value": "dfc" + } + ] + } + }, + "response": [] + }, + { + "name": "fetch-with-multiple-statuses", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs?status=successful,failed", + "host": [ + "{{url}}" + ], + "path": [ + "jobs" + ], + "query": [ + { + "key": "status", + "value": "successful,failed" + } + ] + } + }, + "response": [] + }, + { + "name": "fetch-with-single-tag-texas", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const jobsData = pm.response.json().jobs;", + "pm.test(\"All returned jobs include texas tag\", function () {", + " pm.expect(jobsData.length).to.be.above(0);", + " jobsData.forEach(job => {", + " pm.expect(job.tags).to.include(\"texas\");", + " });", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs?tags=texas", + "host": [ + "{{url}}" + ], + "path": [ + "jobs" + ], + "query": [ + { + "key": "tags", + "value": "texas" + } + ] + } + }, + "response": [] + }, + { + "name": "fetch-with-single-tag-huc984727", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const jobsData = pm.response.json().jobs;", + "pm.test(\"All returned jobs include huc984727 tag\", function () {", + " pm.expect(jobsData.length).to.be.above(0);", + " jobsData.forEach(job => {", + " pm.expect(job.tags).to.include(\"huc984727\");", + " });", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs?tags=huc984727", + "host": [ + "{{url}}" + ], + "path": [ + "jobs" + ], + "query": [ + { + "key": "tags", + "value": "huc984727" + } + ] + } + }, + "response": [] + }, + { + "name": "fetch-with-multiple-tags", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const jobsData = pm.response.json().jobs;", + "pm.test(\"All returned jobs include both texas and huc984727 tags\", function () {", + " pm.expect(jobsData.length).to.be.above(0);", + " jobsData.forEach(job => {", + " pm.expect(job.tags).to.include(\"texas\");", + " pm.expect(job.tags).to.include(\"huc984727\");", + " });", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs?tags=texas,huc984727", + "host": [ + "{{url}}" + ], + "path": [ + "jobs" + ], + "query": [ + { + "key": "tags", + "value": "texas,huc984727" + } + ] + } + }, + "response": [] + }, + { + "name": "check-empty-tags-job", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const job = pm.response.json();", + "pm.test(\"Job with empty tags returns tags as []\", function () {", + " pm.expect(job).to.have.property(\"tags\");", + " pm.expect(job.tags).to.be.an(\"array\");", + " pm.expect(job.tags.length).to.eql(0);", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/{{emptyTagsJobID}}", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + "{{emptyTagsJobID}}" + ] + } + }, + "response": [] + }, + { + "name": "check-no-tags-job", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const job = pm.response.json();", + "pm.test(\"Job without tags returns tags as []\", function () {", + " pm.expect(job).to.have.property(\"tags\");", + " pm.expect(job.tags).to.be.an(\"array\");", + " pm.expect(job.tags.length).to.eql(0);", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/{{noTagsJobID}}", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + "{{noTagsJobID}}" + ] + } + }, + "response": [] + }, + { + "name": "check-tagged-job", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const job = pm.response.json();", + "pm.test(\"Tagged job returns submitted tags\", function () {", + " pm.expect(job).to.have.property(\"tags\");", + " pm.expect(job.tags).to.include(\"texas\");", + " pm.expect(job.tags).to.include(\"huc984727\");", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/{{taggedJobID}}", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + "{{taggedJobID}}" + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test('Status code is 200 ', function(){", + " pm.response.to.have.status(200)", + "});" + ] + } + } + ] + }, + { + "name": "Negatives", + "item": [ + { + "name": "fetch-with-invalid-status", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs?status=invalidStatus", + "host": [ + "{{url}}" + ], + "path": [ + "jobs" + ], + "query": [ + { + "key": "status", + "value": "invalidStatus" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Status code is equal or greater than 400\", function () {", + " pm.expect(pm.response.code).to.be.greaterThan(399)", + "});", + "", + "pm.test(\"response should have message\", function () {", + " pm.response.to.have.jsonBody(\"message\");", + "});" + ] + } + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "pm.test(\"Response time is less than 1000ms\", function () {", + " pm.expect(pm.response.responseTime).to.be.below(1000);", + "});", + "", + "if (pm.response.responseTime > 500) {", + " console.log(`⚠️ Warning: Response time is ${pm.response.responseTime}ms which is greater than 500ms.`);", + "}" + ] + } + } + ] + }, + { + "name": "scheduler", + "item": [ + { + "name": "setup", + "item": [ + { + "name": "register-sleeper-process", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Process registered successfully', function () {", + " pm.expect(pm.response.code).to.be.oneOf([200, 201]);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"info\": {\n \"version\": \"1.0.0\",\n \"id\": \"sleeperScheduler\",\n \"title\": \"Sleeper Scheduler Test\",\n \"description\": \"A test process for scheduler resource contention (supports both sync and async)\",\n \"jobControlOptions\": [\"sync-execute\", \"async-execute\"],\n \"outputTransmission\": [\"value\"]\n },\n \"host\": {\n \"type\": \"subprocess\"\n },\n \"command\": [\"bash\", \"-c\", \"sleep 10 && echo '{\\\"plugin_results\\\": \\\"slept for 10 seconds!\\\"}'\" ],\n \"config\": {\n \"maxResources\": {\n \"cpus\": 4.0,\n \"memory\": 4096\n }\n },\n \"inputs\": [],\n \"outputs\": []\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/sleeperScheduler", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "sleeperScheduler" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "resource-contention", + "item": [ + { + "name": "submit-async-sleeper", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Status code is 201', function () {", + " pm.response.to.have.status(201);", + "});", + "", + "pm.test('Preference-Applied header is set', function () {", + " pm.expect(pm.response.headers.get('Preference-Applied')).to.eql('respond-async');", + "});", + "", + "var resp = pm.response.json();", + "pm.collectionVariables.set('sleeperJobID', resp.jobID);" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Prefer", + "value": "respond-async" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/sleeperScheduler/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "sleeperScheduler", + "execution" + ] + } + }, + "response": [] + }, + { + "name": "check-resources-in-use", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Status code is 200', function () {", + " pm.response.to.have.status(200);", + "});", + "", + "pm.test('Resources are in use or queued', function () {", + " var resources = pm.response.json().resources;", + " var totalCPUs = resources.usedCPUs + resources.queuedCPUs;", + " pm.expect(totalCPUs).to.be.above(0);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/admin/resources", + "host": [ + "{{url}}" + ], + "path": [ + "admin", + "resources" + ] + } + }, + "response": [] + }, + { + "name": "sync-job-fails-when-resources-full", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Sync job returns 503 when resources unavailable', function () {", + " pm.response.to.have.status(503);", + "});", + "", + "pm.test('Response has appropriate error message', function () {", + " var resp = pm.response.json();", + " pm.expect(resp.message).to.include('resources');", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/sleeperScheduler/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "sleeperScheduler", + "execution" + ] + } + }, + "response": [] + }, + { + "name": "sync-job-succeeds-after-wait", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Sync job succeeds when resources available', function () {", + " pm.response.to.have.status(200);", + "});", + "", + "pm.test('Job completed successfully', function () {", + " var resp = pm.response.json();", + " pm.expect(resp.status).to.eql('successful');", + "});" + ], + "type": "text/javascript", + "packages": {}, + "requests": {} + } + }, + { + "listen": "prerequest", + "script": { + "exec": [ + "// Wait 10 seconds for the async sleeper job to complete", + "setTimeout(() => {}, 10000);" + ], + "type": "text/javascript", + "packages": {}, + "requests": {} + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/sleeperScheduler/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "sleeperScheduler", + "execution" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "cleanup", + "item": [ + { + "name": "delete-sleeper-scheduler", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test('Process deleted successfully', function () {", + " pm.response.to.have.status(200);", + "});" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/processes/sleeperScheduler", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "sleeperScheduler" + ] + } + }, + "response": [] + } + ] + } + ], + "description": "Tests for the local scheduler resource management" + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "requests": {}, + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "requests": {}, + "exec": [ + "pm.test(\"response must be valid and have a body\", function () {", + " pm.response.to.be.withBody;", + " pm.response.to.be.json;", + "});" + ] + } + } + ], + "variable": [ + { + "key": "url", + "value": "localhost:5050" + }, + { + "key": "jobID", + "value": "" + }, + { + "key": "sleeperJobID", + "value": "" + }, + { + "key": "execModeJobID", + "value": "" + }, + { + "key": "maxCPUs", + "value": "" + }, + { + "key": "maxMemory", + "value": "" + }, + { + "key": "taggedJobID", + "value": "" + }, + { + "key": "emptyTagsJobID", + "value": "" + }, + { + "key": "noTagsJobID", + "value": "" + } + ] +} From 271b5a1a70895a0835c10d0b4ccfab236a5ee731 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Fri, 13 Mar 2026 10:31:15 -0400 Subject: [PATCH 09/19] Update job number assertion from 11 to 14: --- tests/e2e/tests.postman_collection.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/tests.postman_collection.json b/tests/e2e/tests.postman_collection.json index 8c85c87..65359b2 100644 --- a/tests/e2e/tests.postman_collection.json +++ b/tests/e2e/tests.postman_collection.json @@ -2717,7 +2717,7 @@ "const jobsData = pm.response.json().jobs;", "", "pm.test(\"number of jobs in record test\", function () {", - " pm.expect(jobsData.length).to.eql(11);", + " pm.expect(jobsData.length).to.eql(14);", " });" ], "type": "text/javascript", From 9ab430113449fa412b1e0a9d1d81bf6c25b1c5af Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Fri, 13 Mar 2026 10:42:10 -0400 Subject: [PATCH 10/19] remove omitempty --- api/handlers/handlers.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/handlers/handlers.go b/api/handlers/handlers.go index 777693a..1e22683 100644 --- a/api/handlers/handlers.go +++ b/api/handlers/handlers.go @@ -97,8 +97,8 @@ func prepareResponse(c echo.Context, httpStatus int, renderName string, output i // specs: https://developer.ogc.org/api/processes/index.html#tag/Execute type runRequestBody struct { Inputs map[string]interface{} `json:"inputs"` - Tags []string `json:"tags,omitempty"` - MacID string `json:"macID,omitempty"` + Tags []string `json:"tags"` + MacID string `json:"macID"` } // LandingPage godoc From c2d091bdaa2ed1d3fcf1fcbaa3e150fe9453ecc9 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Fri, 13 Mar 2026 11:29:12 -0400 Subject: [PATCH 11/19] Add tags to job response --- api/handlers/handlers.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api/handlers/handlers.go b/api/handlers/handlers.go index 1e22683..1dc7429 100644 --- a/api/handlers/handlers.go +++ b/api/handlers/handlers.go @@ -39,6 +39,8 @@ type jobResponse struct { ProcessID string `json:"processID,omitempty"` Message string `json:"message,omitempty"` Outputs interface{} `json:"outputs,omitempty"` + Tags []string `json:"tags"` + MacID string `json:"macID,omitempty"` } type link struct { @@ -416,20 +418,28 @@ func (rh *RESTHandler) JobStatusHandler(c echo.Context) (err error) { var jRcrd jobs.JobRecord jobID := c.Param("jobID") + if job, ok := rh.ActiveJobs.Jobs[jobID]; ok { resp := jobResponse{ ProcessID: (*job).ProcessID(), JobID: (*job).JobID(), LastUpdate: (*job).LastUpdate(), Status: (*job).CurrentStatus(), + Tags: []string{}, } return prepareResponse(c, http.StatusOK, "jobStatus", resp) } else if jRcrd, ok, err = rh.DB.GetJob(jobID); ok { + if jRcrd.Tags == nil { + jRcrd.Tags = []string{} + } + resp := jobResponse{ ProcessID: jRcrd.ProcessID, JobID: jRcrd.JobID, LastUpdate: jRcrd.LastUpdate, Status: jRcrd.Status, + Tags: jRcrd.Tags, + MacID: jRcrd.MacID, } return prepareResponse(c, http.StatusOK, "jobStatus", resp) } From 8d61302ac961e07c15e7df899e259d445b94d6aa Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Fri, 13 Mar 2026 11:36:20 -0400 Subject: [PATCH 12/19] add tags to the example postman --- postman.json | 2224 ++++++++++++++++++++++++++------------------------ 1 file changed, 1143 insertions(+), 1081 deletions(-) diff --git a/postman.json b/postman.json index c5ed479..76e3015 100644 --- a/postman.json +++ b/postman.json @@ -1,1082 +1,1144 @@ { - "info": { - "_postman_id": "c4d26f8c-50f5-4f4c-86c9-2aef531bc9a8", - "name": "sepex", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", - "_exporter_id": "36773789" - }, - "item": [ - { - "name": "sync", - "item": [ - { - "name": "execution-cleanFloodPlain", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {\n \"inputFile\": \"/vsis3/texas-glo/clipped-raster-inputs/input_raster.tif\",\n \"outputFile\": \"/vsis3/texas-glo/v2r-clean-outputs/out.tif\"\n }\n}\n\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "cleanFloodPlain" - } - ] - } - }, - "response": [] - }, - { - "name": "pyecho", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"inputs\": {\r\n \"text\": \"hello 🌎\"\r\n }\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/pyecho/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "pyecho", - "execution" - ] - } - }, - "response": [] - }, - { - "name": "subprocessEcho", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript", - "packages": {} - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"inputs\": {}\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/subprocessEcho/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "subprocessEcho", - "execution" - ] - } - }, - "response": [] - }, - { - "name": "pywrite", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"inputs\": {\r\n \"userInput\": \"hello 🌎\",\r\n \"outputFile\": \"hello.txt\"\r\n }\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/pywrite/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - "pywrite", - "execution" - ] - } - }, - "response": [] - }, - { - "name": "execution-dfc", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {\n \"crs\": \"4326\",\n \"points\": [\n [\n -76.491824,\n 37.271065\n ],\n [\n -76.508683,\n 37.264547\n ],\n [\n -76.4920965,\n 37.2735359\n ]\n ],\n \"tile\": \"122\",\n \"epoch\": \"2020\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "dfc" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "var resp = pm.response.json()", - "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" - ] - } - } - ] - }, - { - "name": "async", - "item": [ - { - "name": "dfc", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "var resp = pm.response.json()\r", - "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {\n \"crs\": \"4326\",\n \"points\": [\n [\n -76.491824,\n 37.271065\n ]\n ],\n \"tile\": \"122\",\n \"epoch\": \"2020\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "dfc" - } - ] - } - }, - "response": [] - }, - { - "name": "loggingTest", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "var resp = pm.response.json()\r", - "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "loggingTest" - } - ] - } - }, - "response": [] - }, - { - "name": "execution-aepGrid", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {\n \"aepGridDestination\": \"aep-grid-outputs/output3.tif\",\n \"tile\": \"106\",\n \"epoch\": \"2020\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "aepGrid" - } - ] - } - }, - "response": [] - }, - { - "name": "execution-ogrVersion", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"inputs\": {}\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "ogrVersion" - } - ] - } - }, - "response": [] - }, - { - "name": "aorc-temp-ncdf2zarr-fargate", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{bearer_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"inputs\": {\r\n \"year\": 1979,\r\n \"month\": 6\r\n }\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID/execution", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID", - "execution" - ], - "variable": [ - { - "key": "processID", - "value": "aorc-temp-ncdf2zarr-fargate" - } - ] - } - }, - "response": [] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "var resp = pm.response.json()", - "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" - ] - } - } - ] - }, - { - "name": "callbacks", - "item": [ - { - "name": "status_update", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - }, - { - "listen": "prerequest", - "script": { - "exec": [ - "postman.setGlobalVariable('timestampUtc', (new Date()).toISOString());" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"status\": \"successful\",\n \"updated\": \"{{timestampUtc}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/jobs/:jobID/status", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "status" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "results_update", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"outputs\": [1,2,3]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/jobs/:jobID/results_update", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "results_update" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - } - ] - }, - { - "name": "admin", - "item": [] - }, - { - "name": "conformance", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/conformance", - "host": [ - "{{url}}" - ], - "path": [ - "conformance" - ] - } - }, - "response": [] - }, - { - "name": "landingpage", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/", - "host": [ - "{{url}}" - ], - "path": [ - "" - ] - } - }, - "response": [] - }, - { - "name": "processes", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/processes", - "host": [ - "{{url}}" - ], - "path": [ - "processes" - ] - } - }, - "response": [] - }, - { - "name": "process-describe", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/processes/:processID", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID" - ], - "variable": [ - { - "key": "processID", - "value": "dfc" - } - ] - } - }, - "response": [] - }, - { - "name": "add-process", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"info\": {\r\n \"version\": \"1.0.0\",\r\n \"id\": \"dfc\",\r\n \"title\": \"Depth Frequency Curve\",\r\n \"description\": \"Returns data for depth-frequency curve\",\r\n \"jobControlOptions\": [\"async-execute\"],\r\n \"outputTransmission\": [\"value\"]\r\n },\r\n \"host\": {\r\n \"type\": \"aws-batch\",\r\n \"jobDefinition\": \"process-sandbox:2\",\r\n \"jobQueue\": \"micro-test\"\r\n },\r\n \"command\": [\"python\", \"dfc.py\"],\r\n \"inputs\": [\r\n {\r\n \"id\": \"crs\",\r\n \"title\": \"crs\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 0,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"tile\",\r\n \"title\": \"tile\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"epoch\",\r\n \"title\": \"epoch\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"points\",\r\n \"title\": \"points\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 999999\r\n }\r\n ],\r\n \"outputs\": [\r\n {\r\n \"id\": \"dfc\",\r\n \"title\": \"dfc\",\r\n \"description\": \"\",\r\n \"output\": {\r\n \"transmissionMode\": [\r\n \"value\"\r\n ]\r\n }\r\n }\r\n ]\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID" - ], - "variable": [ - { - "key": "processID", - "value": "dfc" - } - ] - } - }, - "response": [] - }, - { - "name": "update-process", - "request": { - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\r\n \"info\": {\r\n \"version\": \"1.0.0\",\r\n \"id\": \"dfc\",\r\n \"title\": \"Updated Depth Frequency Curve\",\r\n \"description\": \"Returns data for depth-frequency curve\",\r\n \"jobControlOptions\": [\"async-execute\"],\r\n \"outputTransmission\": [\"value\"]\r\n },\r\n \"host\": {\r\n \"type\": \"aws-batch\",\r\n \"jobDefinition\": \"process-sandbox:2\",\r\n \"jobQueue\": \"micro-test\"\r\n },\r\n \"command\": [\"python\", \"dfc.py\"],\r\n \"inputs\": [\r\n {\r\n \"id\": \"crs\",\r\n \"title\": \"crs\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 0,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"tile\",\r\n \"title\": \"tile\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"epoch\",\r\n \"title\": \"epoch\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"points\",\r\n \"title\": \"points\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 999999\r\n }\r\n ],\r\n \"outputs\": [\r\n {\r\n \"id\": \"dfc\",\r\n \"title\": \"dfc\",\r\n \"description\": \"\",\r\n \"output\": {\r\n \"transmissionMode\": [\r\n \"value\"\r\n ]\r\n }\r\n }\r\n ]\r\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{url}}/processes/:processID", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID" - ], - "variable": [ - { - "key": "processID", - "value": "dfc" - } - ] - } - }, - "response": [] - }, - { - "name": "delete-process", - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/processes/:processID", - "host": [ - "{{url}}" - ], - "path": [ - "processes", - ":processID" - ], - "variable": [ - { - "key": "processID", - "value": "dfc" - } - ] - } - }, - "response": [] - }, - { - "name": "jobs", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs", - "host": [ - "{{url}}" - ], - "path": [ - "jobs" - ], - "query": [ - { - "key": "submitter", - "value": "", - "disabled": true - }, - { - "key": "limit", - "value": null, - "disabled": true - }, - { - "key": "offset", - "value": null, - "disabled": true - }, - { - "key": "processid", - "value": null, - "disabled": true - } - ] - } - }, - "response": [] - }, - { - "name": "job-status", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-results", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/results", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "results" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-logs", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/logs", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "logs" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-metadata", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID/metadata", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID", - "metadata" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "job-dismiss", - "request": { - "method": "DELETE", - "header": [], - "url": { - "raw": "{{url}}/jobs/:jobID", - "host": [ - "{{url}}" - ], - "path": [ - "jobs", - ":jobID" - ], - "variable": [ - { - "key": "jobID", - "value": "{{jobID}}" - } - ] - } - }, - "response": [] - }, - { - "name": "login", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "var jsonData = JSON.parse(responseBody);\r", - "postman.setEnvironmentVariable(\"bearer_token\", jsonData.access_token);" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "urlencoded", - "urlencoded": [ - { - "key": "username", - "value": "{{auth_username}}", - "type": "text" - }, - { - "key": "password", - "value": "{{auth_password}}", - "type": "text" - }, - { - "key": "client_id", - "value": "{{auth_client_id}}", - "type": "text" - }, - { - "key": "grant_type", - "value": "password", - "type": "text" - }, - { - "key": "client_secret", - "value": "{{auth_client_secret}}", - "type": "text" - } - ] - }, - "url": { - "raw": "{{auth_url}}/protocol/openid-connect/token", - "host": [ - "{{auth_url}}" - ], - "path": [ - "protocol", - "openid-connect", - "token" - ] - } - }, - "response": [] - }, - { - "name": "service-account-login", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "var jsonData = JSON.parse(responseBody);\r", - "postman.setEnvironmentVariable(\"bearer_token\", jsonData.access_token);" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "urlencoded", - "urlencoded": [ - { - "key": "client_id", - "value": "{{auth_client_id}}", - "type": "text" - }, - { - "key": "grant_type", - "value": "client_credentials", - "type": "text" - }, - { - "key": "client_secret", - "value": "{{auth_client_secret}}", - "type": "text" - } - ] - }, - "url": { - "raw": "{{auth_url}}/protocol/openid-connect/token", - "host": [ - "{{auth_url}}" - ], - "path": [ - "protocol", - "openid-connect", - "token" - ] - } - }, - "response": [] - } - ], - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{bearer_token}}", - "type": "string" - } - ] - }, - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "pm.request.headers.add({", - " key: 'X-SEPEX-User-Email',", - " value: pm.variables.get('auth_username')", - "});" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - } - ], - "variable": [ - { - "key": "jobID", - "value": "" - } - ] -} \ No newline at end of file + "info": { + "_postman_id": "c4d26f8c-50f5-4f4c-86c9-2aef531bc9a8", + "name": "sepex", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "36773789" + }, + "item": [ + { + "name": "sync", + "item": [ + { + "name": "execution-cleanFloodPlain", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {\n \"inputFile\": \"/vsis3/texas-glo/clipped-raster-inputs/input_raster.tif\",\n \"outputFile\": \"/vsis3/texas-glo/v2r-clean-outputs/out.tif\"\n }\n}\n\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "cleanFloodPlain" + } + ] + } + }, + "response": [] + }, + { + "name": "pyecho", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\r\n \"inputs\": {\r\n \"text\": \"hello 🌎\"\r\n }\r\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/pyecho/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "pyecho", + "execution" + ] + } + }, + "response": [] + }, + { + "name": "pyecho-with-tags", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {\n \"text\": \"hello 🌎\"\n },\n \"tags\": [\"texas\", \"huc984727\"]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/pyecho/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "pyecho", + "execution" + ] + } + }, + "response": [] + }, + { + "name": "subprocessEcho", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript", + "packages": {} + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\r\n \"inputs\": {}\r\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/subprocessEcho/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "subprocessEcho", + "execution" + ] + } + }, + "response": [] + }, + { + "name": "pywrite", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\r\n \"inputs\": {\r\n \"userInput\": \"hello 🌎\",\r\n \"outputFile\": \"hello.txt\"\r\n }\r\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/pywrite/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + "pywrite", + "execution" + ] + } + }, + "response": [] + }, + { + "name": "execution-dfc", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {\n \"crs\": \"4326\",\n \"points\": [\n [\n -76.491824,\n 37.271065\n ],\n [\n -76.508683,\n 37.264547\n ],\n [\n -76.4920965,\n 37.2735359\n ]\n ],\n \"tile\": \"122\",\n \"epoch\": \"2020\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "dfc" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var resp = pm.response.json()", + "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" + ] + } + } + ] + }, + { + "name": "async", + "item": [ + { + "name": "dfc", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "var resp = pm.response.json()\r", + "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {\n \"crs\": \"4326\",\n \"points\": [\n [\n -76.491824,\n 37.271065\n ]\n ],\n \"tile\": \"122\",\n \"epoch\": \"2020\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "dfc" + } + ] + } + }, + "response": [] + }, + { + "name": "loggingTest", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "var resp = pm.response.json()\r", + "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "loggingTest" + } + ] + } + }, + "response": [] + }, + { + "name": "execution-aepGrid", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {\n \"aepGridDestination\": \"aep-grid-outputs/output3.tif\",\n \"tile\": \"106\",\n \"epoch\": \"2020\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "aepGrid" + } + ] + } + }, + "response": [] + }, + { + "name": "execution-ogrVersion", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"inputs\": {}\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "ogrVersion" + } + ] + } + }, + "response": [] + }, + { + "name": "aorc-temp-ncdf2zarr-fargate", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{bearer_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\r\n \"inputs\": {\r\n \"year\": 1979,\r\n \"month\": 6\r\n }\r\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID/execution", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID", + "execution" + ], + "variable": [ + { + "key": "processID", + "value": "aorc-temp-ncdf2zarr-fargate" + } + ] + } + }, + "response": [] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var resp = pm.response.json()", + "pm.collectionVariables.set(\"jobID\", resp[\"jobID\"])" + ] + } + } + ] + }, + { + "name": "callbacks", + "item": [ + { + "name": "status_update", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + }, + { + "listen": "prerequest", + "script": { + "exec": [ + "postman.setGlobalVariable('timestampUtc', (new Date()).toISOString());" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"status\": \"successful\",\n \"updated\": \"{{timestampUtc}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/jobs/:jobID/status", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "status" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "results_update", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"outputs\": [1,2,3]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/jobs/:jobID/results_update", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "results_update" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + } + ] + }, + { + "name": "admin", + "item": [] + }, + { + "name": "conformance", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/conformance", + "host": [ + "{{url}}" + ], + "path": [ + "conformance" + ] + } + }, + "response": [] + }, + { + "name": "landingpage", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/", + "host": [ + "{{url}}" + ], + "path": [ + "" + ] + } + }, + "response": [] + }, + { + "name": "processes", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/processes", + "host": [ + "{{url}}" + ], + "path": [ + "processes" + ] + } + }, + "response": [] + }, + { + "name": "process-describe", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/processes/:processID", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID" + ], + "variable": [ + { + "key": "processID", + "value": "dfc" + } + ] + } + }, + "response": [] + }, + { + "name": "add-process", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\r\n \"info\": {\r\n \"version\": \"1.0.0\",\r\n \"id\": \"dfc\",\r\n \"title\": \"Depth Frequency Curve\",\r\n \"description\": \"Returns data for depth-frequency curve\",\r\n \"jobControlOptions\": [\"async-execute\"],\r\n \"outputTransmission\": [\"value\"]\r\n },\r\n \"host\": {\r\n \"type\": \"aws-batch\",\r\n \"jobDefinition\": \"process-sandbox:2\",\r\n \"jobQueue\": \"micro-test\"\r\n },\r\n \"command\": [\"python\", \"dfc.py\"],\r\n \"inputs\": [\r\n {\r\n \"id\": \"crs\",\r\n \"title\": \"crs\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 0,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"tile\",\r\n \"title\": \"tile\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"epoch\",\r\n \"title\": \"epoch\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"points\",\r\n \"title\": \"points\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 999999\r\n }\r\n ],\r\n \"outputs\": [\r\n {\r\n \"id\": \"dfc\",\r\n \"title\": \"dfc\",\r\n \"description\": \"\",\r\n \"output\": {\r\n \"transmissionMode\": [\r\n \"value\"\r\n ]\r\n }\r\n }\r\n ]\r\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID" + ], + "variable": [ + { + "key": "processID", + "value": "dfc" + } + ] + } + }, + "response": [] + }, + { + "name": "update-process", + "request": { + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\r\n \"info\": {\r\n \"version\": \"1.0.0\",\r\n \"id\": \"dfc\",\r\n \"title\": \"Updated Depth Frequency Curve\",\r\n \"description\": \"Returns data for depth-frequency curve\",\r\n \"jobControlOptions\": [\"async-execute\"],\r\n \"outputTransmission\": [\"value\"]\r\n },\r\n \"host\": {\r\n \"type\": \"aws-batch\",\r\n \"jobDefinition\": \"process-sandbox:2\",\r\n \"jobQueue\": \"micro-test\"\r\n },\r\n \"command\": [\"python\", \"dfc.py\"],\r\n \"inputs\": [\r\n {\r\n \"id\": \"crs\",\r\n \"title\": \"crs\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 0,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"tile\",\r\n \"title\": \"tile\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"epoch\",\r\n \"title\": \"epoch\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 1\r\n },\r\n {\r\n \"id\": \"points\",\r\n \"title\": \"points\",\r\n \"description\": \"\",\r\n \"input\": {\r\n \"literalDataDomain\": {\r\n \"dataType\": \"value\",\r\n \"valueDefinition\": {\r\n \"anyValue\": true,\r\n \"possibleValues\": null\r\n }\r\n }\r\n },\r\n \"minOccurs\": 1,\r\n \"maxOccurs\": 999999\r\n }\r\n ],\r\n \"outputs\": [\r\n {\r\n \"id\": \"dfc\",\r\n \"title\": \"dfc\",\r\n \"description\": \"\",\r\n \"output\": {\r\n \"transmissionMode\": [\r\n \"value\"\r\n ]\r\n }\r\n }\r\n ]\r\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{url}}/processes/:processID", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID" + ], + "variable": [ + { + "key": "processID", + "value": "dfc" + } + ] + } + }, + "response": [] + }, + { + "name": "delete-process", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/processes/:processID", + "host": [ + "{{url}}" + ], + "path": [ + "processes", + ":processID" + ], + "variable": [ + { + "key": "processID", + "value": "dfc" + } + ] + } + }, + "response": [] + }, + { + "name": "jobs", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs", + "host": [ + "{{url}}" + ], + "path": [ + "jobs" + ], + "query": [ + { + "key": "submitter", + "value": "", + "disabled": true + }, + { + "key": "limit", + "value": null, + "disabled": true + }, + { + "key": "offset", + "value": null, + "disabled": true + }, + { + "key": "processid", + "value": null, + "disabled": true + } + ] + } + }, + "response": [] + }, + { + "name": "jobs-filter-by-tags", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs?tags=texas,huc984727", + "host": [ + "{{url}}" + ], + "path": [ + "jobs" + ], + "query": [ + { + "key": "tags", + "value": "texas,huc984727" + } + ] + } + }, + "response": [] + }, + { + "name": "job-status", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-results", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/results", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "results" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-logs", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/logs", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "logs" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-metadata", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID/metadata", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID", + "metadata" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "job-dismiss", + "request": { + "method": "DELETE", + "header": [], + "url": { + "raw": "{{url}}/jobs/:jobID", + "host": [ + "{{url}}" + ], + "path": [ + "jobs", + ":jobID" + ], + "variable": [ + { + "key": "jobID", + "value": "{{jobID}}" + } + ] + } + }, + "response": [] + }, + { + "name": "login", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "var jsonData = JSON.parse(responseBody);\r", + "postman.setEnvironmentVariable(\"bearer_token\", jsonData.access_token);" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "urlencoded", + "urlencoded": [ + { + "key": "username", + "value": "{{auth_username}}", + "type": "text" + }, + { + "key": "password", + "value": "{{auth_password}}", + "type": "text" + }, + { + "key": "client_id", + "value": "{{auth_client_id}}", + "type": "text" + }, + { + "key": "grant_type", + "value": "password", + "type": "text" + }, + { + "key": "client_secret", + "value": "{{auth_client_secret}}", + "type": "text" + } + ] + }, + "url": { + "raw": "{{auth_url}}/protocol/openid-connect/token", + "host": [ + "{{auth_url}}" + ], + "path": [ + "protocol", + "openid-connect", + "token" + ] + } + }, + "response": [] + }, + { + "name": "service-account-login", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "var jsonData = JSON.parse(responseBody);\r", + "postman.setEnvironmentVariable(\"bearer_token\", jsonData.access_token);" + ], + "type": "text/javascript" + } + } + ], + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "urlencoded", + "urlencoded": [ + { + "key": "client_id", + "value": "{{auth_client_id}}", + "type": "text" + }, + { + "key": "grant_type", + "value": "client_credentials", + "type": "text" + }, + { + "key": "client_secret", + "value": "{{auth_client_secret}}", + "type": "text" + } + ] + }, + "url": { + "raw": "{{auth_url}}/protocol/openid-connect/token", + "host": [ + "{{auth_url}}" + ], + "path": [ + "protocol", + "openid-connect", + "token" + ] + } + }, + "response": [] + } + ], + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{bearer_token}}", + "type": "string" + } + ] + }, + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "pm.request.headers.add({", + " key: 'X-SEPEX-User-Email',", + " value: pm.variables.get('auth_username')", + "});" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "" + ] + } + } + ], + "variable": [ + { + "key": "jobID", + "value": "" + } + ] +} From 7ddbfc9734af79a2e55526dcd8bd032131abc098 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Tue, 17 Mar 2026 12:58:56 -0400 Subject: [PATCH 13/19] remove MAC ID --- api/handlers/handlers.go | 9 +-------- api/jobs/aws_batch_jobs.go | 3 +-- api/jobs/database.go | 4 ++-- api/jobs/database_postgres.go | 18 ++++++++---------- api/jobs/database_sqlite.go | 19 +++++++++---------- api/jobs/docker_jobs.go | 3 +-- api/jobs/jobs.go | 1 - api/jobs/subprocess_jobs.go | 3 +-- api/views/jobs.html | 2 -- 9 files changed, 23 insertions(+), 39 deletions(-) diff --git a/api/handlers/handlers.go b/api/handlers/handlers.go index 1dc7429..dec552c 100644 --- a/api/handlers/handlers.go +++ b/api/handlers/handlers.go @@ -40,7 +40,6 @@ type jobResponse struct { Message string `json:"message,omitempty"` Outputs interface{} `json:"outputs,omitempty"` Tags []string `json:"tags"` - MacID string `json:"macID,omitempty"` } type link struct { @@ -100,7 +99,6 @@ func prepareResponse(c echo.Context, httpStatus int, renderName string, output i type runRequestBody struct { Inputs map[string]interface{} `json:"inputs"` Tags []string `json:"tags"` - MacID string `json:"macID"` } // LandingPage godoc @@ -254,7 +252,6 @@ func (rh *RESTHandler) Execution(c echo.Context) error { DB: rh.DB, DoneChan: rh.MessageQueue.JobDone, Tags: params.Tags, - MacID: params.MacID, ResourcePool: rh.ResourcePool, IsSync: mode == "sync-execute", } @@ -275,7 +272,6 @@ func (rh *RESTHandler) Execution(c echo.Context) error { DB: rh.DB, DoneChan: rh.MessageQueue.JobDone, Tags: params.Tags, - MacID: params.MacID, } case "subprocess": @@ -291,7 +287,6 @@ func (rh *RESTHandler) Execution(c echo.Context) error { DB: rh.DB, DoneChan: rh.MessageQueue.JobDone, Tags: params.Tags, - MacID: params.MacID, ResourcePool: rh.ResourcePool, IsSync: mode == "sync-execute", } @@ -439,7 +434,6 @@ func (rh *RESTHandler) JobStatusHandler(c echo.Context) (err error) { LastUpdate: jRcrd.LastUpdate, Status: jRcrd.Status, Tags: jRcrd.Tags, - MacID: jRcrd.MacID, } return prepareResponse(c, http.StatusOK, "jobStatus", resp) } @@ -643,7 +637,6 @@ func (rh *RESTHandler) ListJobsHandler(c echo.Context) error { if tagsParam != "" { tagsList = strings.Split(tagsParam, ",") } - macIDParam := c.QueryParam("macID") var processIDList []string if processIDs != "" { @@ -687,7 +680,7 @@ func (rh *RESTHandler) ListJobsHandler(c echo.Context) error { offset = 0 } - result, err := rh.DB.GetJobs(limit, offset, processIDList, statusList, submittersList, tagsList, macIDParam) + result, err := rh.DB.GetJobs(limit, offset, processIDList, statusList, submittersList, tagsList) if err != nil { output := errResponse{HTTPStatus: http.StatusInternalServerError, Message: err.Error()} return prepareResponse(c, http.StatusNotFound, "error", output) diff --git a/api/jobs/aws_batch_jobs.go b/api/jobs/aws_batch_jobs.go index 8865c78..60ba6f5 100644 --- a/api/jobs/aws_batch_jobs.go +++ b/api/jobs/aws_batch_jobs.go @@ -39,7 +39,6 @@ type AWSBatchJob struct { Status string `json:"status"` // results interface{} Tags []string `json:"tags"` - MacID string `json:"macID"` logger *log.Logger logFile *os.File @@ -274,7 +273,7 @@ func (j *AWSBatchJob) Create() error { j.batchContext = batchContext // At this point job is ready to be added to database - err = j.DB.addJob(j.UUID, "accepted", "", "aws-batch", j.AWSBatchID, j.ProcessName, j.Submitter, j.Tags, j.MacID, time.Now()) + err = j.DB.addJob(j.UUID, "accepted", "", "aws-batch", j.AWSBatchID, j.ProcessName, j.Submitter, j.Tags, time.Now()) if err != nil { j.ctxCancel() return err diff --git a/api/jobs/database.go b/api/jobs/database.go index d993e96..74f22f0 100644 --- a/api/jobs/database.go +++ b/api/jobs/database.go @@ -8,7 +8,7 @@ import ( // Database interface abstracts database operations type Database interface { - addJob(jid, status, mode, host, hostJobID, processID, submitter string, tags []string, macID string, updated time.Time) error + addJob(jid, status, mode, host, hostJobID, processID, submitter string, tags []string, updated time.Time) error updateJobRecord(jid, status string, now time.Time) error updateJobHostId(jid, hostJobID string) error @@ -16,7 +16,7 @@ type Database interface { GetJob(jid string) (JobRecord, bool, error) CheckJobExist(jid string) (bool, error) - GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string, macID string) ([]JobRecord, error) + GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string) ([]JobRecord, error) Close() error } diff --git a/api/jobs/database_postgres.go b/api/jobs/database_postgres.go index 2978ecb..1863364 100644 --- a/api/jobs/database_postgres.go +++ b/api/jobs/database_postgres.go @@ -54,8 +54,7 @@ func (postgresDB *PostgresDB) createTables() error { host_job_id TEXT NOT NULL DEFAULT '', process_id TEXT NOT NULL, submitter TEXT NOT NULL DEFAULT '', - tags TEXT[] NOT NULL DEFAULT '{}', - macID TEXT NOT NULL DEFAULT '' + tags TEXT[] NOT NULL DEFAULT '{}' ); CREATE INDEX IF NOT EXISTS idx_jobs_updated ON jobs(updated); CREATE INDEX IF NOT EXISTS idx_jobs_process_id ON jobs(process_id); @@ -80,9 +79,9 @@ func (postgresDB *PostgresDB) createTables() error { } // AddJob adds a new job to the database -func (db *PostgresDB) addJob(jid, status, mode, host, hostJobID, processID, submitter string, tags []string, macID string, updated time.Time) error { - query := `INSERT INTO jobs (id, status, updated, mode, host, host_job_id, process_id, submitter, tags, macID) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)` - _, err := db.Handle.Exec(query, jid, status, updated, mode, host, hostJobID, processID, submitter, pq.Array(tags), macID) +func (db *PostgresDB) addJob(jid, status, mode, host, hostJobID, processID, submitter string, tags []string, updated time.Time) error { + query := `INSERT INTO jobs (id, status, updated, mode, host, host_job_id, process_id, submitter, tags) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)` + _, err := db.Handle.Exec(query, jid, status, updated, mode, host, hostJobID, processID, submitter, pq.Array(tags)) return err } @@ -95,7 +94,7 @@ func (db *PostgresDB) updateJobRecord(jid, status string, now time.Time) error { // GetJob retrieves a job record by id func (db *PostgresDB) GetJob(jid string) (JobRecord, bool, error) { - query := `SELECT id, status, updated, mode, host, host_job_id, process_id, submitter, tags, macID FROM jobs WHERE id = $1` + query := `SELECT id, status, updated, mode, host, host_job_id, process_id, submitter, tags FROM jobs WHERE id = $1` var jr JobRecord err := db.Handle.QueryRow(query, jid).Scan( &jr.JobID, @@ -107,7 +106,6 @@ func (db *PostgresDB) GetJob(jid string) (JobRecord, bool, error) { &jr.ProcessID, &jr.Submitter, pq.Array(&jr.Tags), - &jr.MacID, ) if err != nil { if err == sql.ErrNoRows { @@ -133,8 +131,8 @@ func (db *PostgresDB) CheckJobExist(jid string) (bool, error) { } // Assumes query parameters are valid -func (pgDB *PostgresDB) GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string, macID string) ([]JobRecord, error) { - baseQuery := `SELECT id, status, updated, process_id, submitter, tags, macID FROM jobs` +func (pgDB *PostgresDB) GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string) ([]JobRecord, error) { + baseQuery := `SELECT id, status, updated, process_id, submitter, tags FROM jobs` whereClauses := []string{} args := []interface{}{} argIndex := 1 @@ -195,7 +193,7 @@ func (pgDB *PostgresDB) GetJobs(limit, offset int, processIDs, statuses, submitt res := []JobRecord{} for rows.Next() { var r JobRecord - if err := rows.Scan(&r.JobID, &r.Status, &r.LastUpdate, &r.ProcessID, &r.Submitter, pq.Array(&r.Tags), &r.MacID); err != nil { + if err := rows.Scan(&r.JobID, &r.Status, &r.LastUpdate, &r.ProcessID, &r.Submitter, pq.Array(&r.Tags)); err != nil { return nil, err } res = append(res, r) diff --git a/api/jobs/database_sqlite.go b/api/jobs/database_sqlite.go index 23b474f..da2866c 100644 --- a/api/jobs/database_sqlite.go +++ b/api/jobs/database_sqlite.go @@ -80,8 +80,7 @@ CREATE TABLE IF NOT EXISTS jobs ( host_job_id TEXT NOT NULL DEFAULT '', process_id TEXT NOT NULL, submitter TEXT NOT NULL DEFAULT '', - tags TEXT NOT NULL DEFAULT '', - macID TEXT NOT NULL DEFAULT '' + tags TEXT NOT NULL DEFAULT '' ); CREATE INDEX IF NOT EXISTS idx_jobs_updated ON jobs(updated); @@ -97,9 +96,9 @@ CREATE TABLE IF NOT EXISTS jobs ( } // Add job to the database. Will return error if job exist. -func (sqliteDB *SQLiteDB) addJob(jid, status, mode, host, hostJobID, processID, submitter string, tags []string, macID string, updated time.Time) error { - query := `INSERT INTO jobs (id, status, updated, mode, host, host_job_id, process_id, submitter, tags, macID) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - _, err := sqliteDB.Handle.Exec(query, jid, status, updated, mode, host, hostJobID, processID, submitter, joinTags(tags), macID) +func (sqliteDB *SQLiteDB) addJob(jid, status, mode, host, hostJobID, processID, submitter string, tags []string, updated time.Time) error { + query := `INSERT INTO jobs (id, status, updated, mode, host, host_job_id, process_id, submitter, tags) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` + _, err := sqliteDB.Handle.Exec(query, jid, status, updated, mode, host, hostJobID, processID, submitter, joinTags(tags)) return err } @@ -124,11 +123,11 @@ func (sqliteDB *SQLiteDB) updateJobRecord(jid, status string, now time.Time) err // If job do not exists, or error encountered bool would be false. // Similar behavior as key exist in hashmap. func (sqliteDB *SQLiteDB) GetJob(jid string) (JobRecord, bool, error) { - query := `SELECT id, status, updated, mode, host, host_job_id, process_id, submitter, tags, macID FROM jobs WHERE id = ?` + query := `SELECT id, status, updated, mode, host, host_job_id, process_id, submitter, tags FROM jobs WHERE id = ?` jr := JobRecord{} var tagsStr string row := sqliteDB.Handle.QueryRow(query, jid) - err := row.Scan(&jr.JobID, &jr.Status, &jr.LastUpdate, &jr.Mode, &jr.Host, &jr.HostJobID, &jr.ProcessID, &jr.Submitter, &tagsStr, &jr.MacID) + err := row.Scan(&jr.JobID, &jr.Status, &jr.LastUpdate, &jr.Mode, &jr.Host, &jr.HostJobID, &jr.ProcessID, &jr.Submitter, &tagsStr) if err != nil { if err == sql.ErrNoRows { return JobRecord{}, false, nil @@ -159,8 +158,8 @@ func (sqliteDB *SQLiteDB) CheckJobExist(jid string) (bool, error) { } // Assumes query parameters are valid -func (sqliteDB *SQLiteDB) GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string, macID string) ([]JobRecord, error) { - baseQuery := `SELECT id, status, updated, process_id, submitter, tags, macID FROM jobs` +func (sqliteDB *SQLiteDB) GetJobs(limit, offset int, processIDs, statuses, submitters, tags []string) ([]JobRecord, error) { + baseQuery := `SELECT id, status, updated, process_id, submitter, tags FROM jobs` whereClauses := []string{} args := []interface{}{} @@ -209,7 +208,7 @@ func (sqliteDB *SQLiteDB) GetJobs(limit, offset int, processIDs, statuses, submi for rows.Next() { var r JobRecord var tagsStr string - if err := rows.Scan(&r.JobID, &r.Status, &r.LastUpdate, &r.ProcessID, &r.Submitter, &tagsStr, &r.MacID); err != nil { + if err := rows.Scan(&r.JobID, &r.Status, &r.LastUpdate, &r.ProcessID, &r.Submitter, &tagsStr); err != nil { return nil, err } r.Tags = splitTags(tagsStr) diff --git a/api/jobs/docker_jobs.go b/api/jobs/docker_jobs.go index 1b9bdf4..89b5a4b 100644 --- a/api/jobs/docker_jobs.go +++ b/api/jobs/docker_jobs.go @@ -40,7 +40,6 @@ type DockerJob struct { Tags []string `json:"tags"` logger *log.Logger logFile *os.File - MacID string `json:"macID"` Resources DB Database @@ -242,7 +241,7 @@ func (j *DockerJob) Create() error { j.ctxCancel = cancelFunc // At this point job is ready to be added to database - err = j.DB.addJob(j.UUID, "accepted", "", "docker", "", j.ProcessName, j.Submitter, j.Tags, j.MacID, time.Now()) + err = j.DB.addJob(j.UUID, "accepted", "", "docker", "", j.ProcessName, j.Submitter, j.Tags, time.Now()) if err != nil { j.ctxCancel() return err diff --git a/api/jobs/jobs.go b/api/jobs/jobs.go index 97f47d8..5ca857c 100644 --- a/api/jobs/jobs.go +++ b/api/jobs/jobs.go @@ -90,7 +90,6 @@ type JobRecord struct { Mode string `json:"mode,omitempty"` Submitter string `json:"submitter"` Tags []string `json:"tags"` - MacID string `json:"macID"` } type LogEntry struct { diff --git a/api/jobs/subprocess_jobs.go b/api/jobs/subprocess_jobs.go index e49ded0..9096d30 100644 --- a/api/jobs/subprocess_jobs.go +++ b/api/jobs/subprocess_jobs.go @@ -35,7 +35,6 @@ type SubprocessJob struct { UpdateTime time.Time Status string `json:"status"` Tags []string `json:"tags"` - MacID string `json:"macID"` execCmd *exec.Cmd logger *log.Logger @@ -189,7 +188,7 @@ func (j *SubprocessJob) Create() error { j.ctxCancel = cancelFunc // At this point job is ready to be added to database - err = j.DB.addJob(j.UUID, "accepted", "", "subprocess", "", j.ProcessName, j.Submitter, j.Tags, j.MacID, time.Now()) + err = j.DB.addJob(j.UUID, "accepted", "", "subprocess", "", j.ProcessName, j.Submitter, j.Tags, time.Now()) if err != nil { j.ctxCancel() diff --git a/api/views/jobs.html b/api/views/jobs.html index da650a5..7b4d6e4 100644 --- a/api/views/jobs.html +++ b/api/views/jobs.html @@ -20,7 +20,6 @@

Jobs List

Status ProcessID Tags - MacID Submitter Updated @@ -47,7 +46,6 @@

Jobs List

{{.ProcessID}} {{range .Tags}}{{.}} {{end}} - {{.MacID}} {{.Submitter}} {{.LastUpdate.Format "2006-01-02 15:04:05 MST"}} From ba30a3f6fb74a0173821edc1d338444797a9b893 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Thu, 19 Mar 2026 10:49:19 -0400 Subject: [PATCH 14/19] remove mac id from doc --- CHANGELOG.md | 4 +--- DEV_GUIDE.md | 2 -- api/jobs/database_postgres.go | 3 --- 3 files changed, 1 insertion(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67b50ef..25a2697 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,12 +16,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm #### POST /processes/:processID/execution - Added optional `tags` array to execution payload. Tags are stored with the job record and can be used to filter jobs when querying `/jobs`. -- Added optional `macID` string to execution payload to associate jobs with a specific client or machine. #### GET /jobs -- Added `macID` query parameter to filter jobs by machine identifier. -- Job responses now include `tags` and `macID` fields. +- Job responses now includes the `tags` field. ## [0.2.2] - 2025-2-28 diff --git a/DEV_GUIDE.md b/DEV_GUIDE.md index 94766eb..0945ccd 100644 --- a/DEV_GUIDE.md +++ b/DEV_GUIDE.md @@ -31,7 +31,6 @@ - If `"inputs": {}` in `/execution` payload, nothing will be appended to process commands. This allows running processes that do not have any inputs. - `"tags"` (optional) can be included as an array of strings in the `/execution` payload. Tags are stored with the job record and can be used to filter jobs when querying `/jobs`. If omitted, the job will be created with an empty tag list. -- `"macID"` (optional) can be included as a string in the `/execution` payload. This value is stored with the job record and can be used to associate jobs with a specific machine or client and filter jobs via the `/jobs` endpoint. Example payload: @@ -41,7 +40,6 @@ Example payload: "text": "Hello World" }, "tags": ["example", "test"], - "macID": "machine-01" } ``` diff --git a/api/jobs/database_postgres.go b/api/jobs/database_postgres.go index 1863364..527e345 100644 --- a/api/jobs/database_postgres.go +++ b/api/jobs/database_postgres.go @@ -72,9 +72,6 @@ func (postgresDB *PostgresDB) createTables() error { if _, err := postgresDB.Handle.Exec(`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS tags TEXT[] NOT NULL DEFAULT '{}';`); err != nil { return fmt.Errorf("error adding tags column: %s", err) } - if _, err := postgresDB.Handle.Exec(`ALTER TABLE jobs ADD COLUMN IF NOT EXISTS macID TEXT NOT NULL DEFAULT '';`); err != nil { - return fmt.Errorf("error adding macID column: %s", err) - } return nil } From ae663a9b6a5b2c235a08da0c908674fbf1530181 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Fri, 20 Mar 2026 12:52:54 -0400 Subject: [PATCH 15/19] preform url-validation on tags --- api/handlers/handlers.go | 5 +++++ api/utils/utils.go | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/api/handlers/handlers.go b/api/handlers/handlers.go index dec552c..f31e26b 100644 --- a/api/handlers/handlers.go +++ b/api/handlers/handlers.go @@ -196,6 +196,11 @@ func (rh *RESTHandler) Execution(c echo.Context) error { params.Tags = []string{} // default to empty if not provided } + err = utils.ValidateTags(params.Tags) + if err != nil { + return c.JSON(http.StatusBadRequest, errResponse{Message: err.Error()}) + } + err = p.VerifyInputs(params.Inputs) if err != nil { return c.JSON(http.StatusBadRequest, errResponse{Message: err.Error()}) diff --git a/api/utils/utils.go b/api/utils/utils.go index 1b08961..6fd99fa 100644 --- a/api/utils/utils.go +++ b/api/utils/utils.go @@ -4,8 +4,11 @@ import ( "bufio" "bytes" "encoding/json" + "fmt" "io" "os" + "regexp" + "strings" "time" "github.com/aws/aws-sdk-go/aws" @@ -134,3 +137,20 @@ func GetS3LinesData(key string, svc *s3.S3) ([]string, error) { return lines, nil } + +var validTagPattern = regexp.MustCompile(`^[A-Za-z0-9._\-]+$`) + +func ValidateTags(tags []string) error { + for _, tag := range tags { + tag = strings.TrimSpace(tag) + + if tag == "" { + return fmt.Errorf("tags cannot contain empty values") + } + + if !validTagPattern.MatchString(tag) { + return fmt.Errorf("tag %q contains unsupported characters. Allowed characters are letters, numbers, '.', '-', and '_'", tag) + } + } + return nil +} From 8ebf009f5a1e13984100fdc8786f716be8b1bba1 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Fri, 20 Mar 2026 12:53:19 -0400 Subject: [PATCH 16/19] add test for prefix search of tags --- tests/e2e/tests.postman_collection.json | 80 +++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/tests/e2e/tests.postman_collection.json b/tests/e2e/tests.postman_collection.json index 65359b2..88b2578 100644 --- a/tests/e2e/tests.postman_collection.json +++ b/tests/e2e/tests.postman_collection.json @@ -3080,6 +3080,86 @@ } }, "response": [] + }, + { + "name": "fetch-with-tag-prefix-tes", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const jobsData = pm.response.json().jobs;", + "pm.test(\"Searching tags by prefix tes returns texas-tagged jobs\", function () {", + " pm.expect(jobsData.length).to.be.above(0);", + " jobsData.forEach(job => {", + " pm.expect(job.tags.some(tag => tag.includes(\"tes\"))).to.eql(true);", + " });", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs?tags=tes", + "host": [ + "{{url}}" + ], + "path": [ + "jobs" + ], + "query": [ + { + "key": "tags", + "value": "tes" + } + ] + } + }, + "response": [] + }, + { + "name": "fetch-with-tag-substring-xas", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "const jobsData = pm.response.json().jobs;", + "pm.test(\"Searching tags by substring xas returns texas-tagged jobs\", function () {", + " pm.expect(jobsData.length).to.be.above(0);", + " jobsData.forEach(job => {", + " pm.expect(job.tags.some(tag => tag.includes(\"xas\"))).to.eql(true);", + " });", + "});" + ] + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{url}}/jobs?tags=xas", + "host": [ + "{{url}}" + ], + "path": [ + "jobs" + ], + "query": [ + { + "key": "tags", + "value": "xas" + } + ] + } + }, + "response": [] } ], "event": [ From 4a1df4070b2b452028b86db1919bd812a3a2002e Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Fri, 20 Mar 2026 13:00:23 -0400 Subject: [PATCH 17/19] correct tes to tex --- run_e2e_local.sh | 101 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100755 run_e2e_local.sh diff --git a/run_e2e_local.sh b/run_e2e_local.sh new file mode 100755 index 0000000..82675a0 --- /dev/null +++ b/run_e2e_local.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- Create env file (same as GH Actions) --- +cat > .env <<'EOF' +API_NAME='github-testing-sepex' +STORAGE_SERVICE='minio' +STORAGE_BUCKET='sepex-storage' +STORAGE_METADATA_PREFIX='metadata' +STORAGE_RESULTS_PREFIX='results' +STORAGE_LOGS_PREFIX='logs' + +PLUGINS_LOAD_DIR='plugins' +PLUGINS_DIR='/.data/plugins' +TMP_JOB_LOGS_DIR='/.data/tmp/logs' + +MAX_LOCAL_CPUS=4 +MAX_LOCAL_MEMORY=4096 + +MINIO_ACCESS_KEY_ID=user +MINIO_SECRET_ACCESS_KEY=password +MINIO_S3_ENDPOINT=http://minio:9000 +MINIO_S3_REGION='us-east-1' +MINIO_ROOT_USER=user +MINIO_ROOT_PASSWORD=password + +DB_SERVICE='postgres' +POSTGRES_CONN_STRING='postgres://user:password@postgres:5432/db?sslmode=disable' +POSTGRES_PASSWORD='password' +POSTGRES_USER='user' +POSTGRES_DB='db' +PG_LOG_CHECKPOINTS='off' + +# Optional (only needed if your tests hit aws-batch paths) +# AWS_ACCESS_KEY_ID=... +# AWS_SECRET_ACCESS_KEY=... +# AWS_REGION=... + +BATCH_LOG_STREAM_GROUP='/aws/batch/job' + +PYWRITE_MINIO_ACCESS_KEY_ID='user' +PYWRITE_MINIO_SECRET_ACCESS_KEY='password' +PYWRITE_MINIO_S3_ENDPOINT='http://minio:9000' +PYWRITE_MINIO_S3_REGION='us-east-1' +PYWRITE_MINIO_S3_BUCKET='sepex-storage' +EOF + +# --- Build plugin example images (same as GH Actions) --- +( cd plugin-examples && chmod +x build.sh && ./build.sh ) & + +# --- Build compose stack --- +docker compose -f docker-compose.prod.yml build + +# --- Network (ignore error if already exists) --- +docker network create process_api_net >/dev/null 2>&1 || true + +# --- Run stack --- +docker compose -f docker-compose.prod.yml up -d + +# --- Create bucket in minio --- +docker run \ + --network process_api_net \ + -e MINIO_ROOT_USER=user \ + -e MINIO_ROOT_PASSWORD=password \ + -e STORAGE_BUCKET=sepex-storage \ + --entrypoint /bin/sh \ + minio/mc:RELEASE.2023-08-18T21-57-55Z \ + -c "mc alias set myminio http://minio:9000 \$MINIO_ROOT_USER \$MINIO_ROOT_PASSWORD && mc mb -p myminio/\${STORAGE_BUCKET} || true" + +# --- Wait for API --- +attempts=0 +max_attempts=12 +until curl -fsS http://localhost:80 >/dev/null 2>&1; do + attempts=$((attempts+1)) + if [ "$attempts" -ge "$max_attempts" ]; then + echo "API not ready after $max_attempts attempts" + docker compose logs + exit 1 + fi + echo "Waiting for API server... attempt $attempts" + sleep 10 +done +echo "API server is ready!" + +# --- Run newman tests (use host networking like GH Actions) --- +docker run --network="host" \ + -v "$PWD/tests/e2e:/etc/newman" \ + postman/newman:5.3.1-alpine \ + run tests.postman_collection.json \ + --env-var "url=localhost:80" \ + --reporters cli \ + --bail \ + --color on + +# --- Logs (helpful locally too) --- +echo "---- docker compose logs ----" +docker compose logs || true +if [ -f .data/api/logs/api.jsonl ]; then + echo "---- .data/api/logs/api.jsonl ----" + cat .data/api/logs/api.jsonl || true +fi From 181ce8245f0a043cb7c4a6cabb70eade752d04e3 Mon Sep 17 00:00:00 2001 From: Anton Kopti Date: Fri, 20 Mar 2026 13:05:11 -0400 Subject: [PATCH 18/19] fix the tex prefix test --- tests/e2e/tests.postman_collection.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/tests.postman_collection.json b/tests/e2e/tests.postman_collection.json index 88b2578..b1d09bd 100644 --- a/tests/e2e/tests.postman_collection.json +++ b/tests/e2e/tests.postman_collection.json @@ -3082,7 +3082,7 @@ "response": [] }, { - "name": "fetch-with-tag-prefix-tes", + "name": "fetch-with-tag-prefix-tex", "event": [ { "listen": "test", @@ -3090,10 +3090,10 @@ "type": "text/javascript", "exec": [ "const jobsData = pm.response.json().jobs;", - "pm.test(\"Searching tags by prefix tes returns texas-tagged jobs\", function () {", + "pm.test(\"Searching tags by prefix tex returns texas-tagged jobs\", function () {", " pm.expect(jobsData.length).to.be.above(0);", " jobsData.forEach(job => {", - " pm.expect(job.tags.some(tag => tag.includes(\"tes\"))).to.eql(true);", + " pm.expect(job.tags.some(tag => tag.includes(\"tex\"))).to.eql(true);", " });", "});" ] @@ -3104,7 +3104,7 @@ "method": "GET", "header": [], "url": { - "raw": "{{url}}/jobs?tags=tes", + "raw": "{{url}}/jobs?tags=tex", "host": [ "{{url}}" ], @@ -3114,7 +3114,7 @@ "query": [ { "key": "tags", - "value": "tes" + "value": "tex" } ] } From a27c093a64504fcf9d90aa01e143813a08d80830 Mon Sep 17 00:00:00 2001 From: ar-siddiqui Date: Wed, 1 Apr 2026 18:03:17 -0400 Subject: [PATCH 19/19] PR-117 review updates --- CHANGELOG.md | 2 -- DEV_GUIDE.md | 2 +- api/handlers/handlers.go | 21 +++++++++++++++------ api/jobs/aws_batch_jobs.go | 4 ++++ api/jobs/database_postgres.go | 6 +++--- api/jobs/database_sqlite.go | 5 +++-- api/jobs/docker_jobs.go | 4 ++++ api/jobs/jobs.go | 1 + api/jobs/subprocess_jobs.go | 4 ++++ api/public/css/main.css | 9 +++++++++ api/utils/utils.go | 11 +++++++---- api/views/jobStatus.html | 1 + api/views/jobs.html | 1 + api/views/statusTable.html | 4 ++++ api/views/tagScripts.html | 14 ++++++++++++++ tests/e2e/tests.postman_collection.json | 9 +++------ 16 files changed, 74 insertions(+), 24 deletions(-) create mode 100644 api/views/tagScripts.html diff --git a/CHANGELOG.md b/CHANGELOG.md index 25a2697..d4912bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,6 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## Unreleased -## [0.2.3] - 2026-3-05 - ### API #### POST /processes/:processID/execution diff --git a/DEV_GUIDE.md b/DEV_GUIDE.md index 0945ccd..7c02e32 100644 --- a/DEV_GUIDE.md +++ b/DEV_GUIDE.md @@ -39,7 +39,7 @@ Example payload: "inputs": { "text": "Hello World" }, - "tags": ["example", "test"], + "tags": ["example", "test"] } ``` diff --git a/api/handlers/handlers.go b/api/handlers/handlers.go index f31e26b..f9ffcf4 100644 --- a/api/handlers/handlers.go +++ b/api/handlers/handlers.go @@ -196,7 +196,7 @@ func (rh *RESTHandler) Execution(c echo.Context) error { params.Tags = []string{} // default to empty if not provided } - err = utils.ValidateTags(params.Tags) + err = utils.SanitizeTags(params.Tags) if err != nil { return c.JSON(http.StatusBadRequest, errResponse{Message: err.Error()}) } @@ -317,7 +317,7 @@ func (rh *RESTHandler) Execution(c echo.Context) error { c.Response().Header().Set("Preference-Applied", modeResult.PreferenceApplied) } - resp := jobResponse{ProcessID: j.ProcessID(), Type: "process", JobID: jobID, Status: j.CurrentStatus()} + resp := jobResponse{ProcessID: j.ProcessID(), Type: "process", JobID: jobID, Status: j.CurrentStatus(), Tags: params.Tags} switch mode { case "sync-execute": j.Run() @@ -420,12 +420,16 @@ func (rh *RESTHandler) JobStatusHandler(c echo.Context) (err error) { jobID := c.Param("jobID") if job, ok := rh.ActiveJobs.Jobs[jobID]; ok { + tags := (*job).TAGS() + if tags == nil { + tags = []string{} + } resp := jobResponse{ ProcessID: (*job).ProcessID(), JobID: (*job).JobID(), LastUpdate: (*job).LastUpdate(), Status: (*job).CurrentStatus(), - Tags: []string{}, + Tags: tags, } return prepareResponse(c, http.StatusOK, "jobStatus", resp) } else if jRcrd, ok, err = rh.DB.GetJob(jobID); ok { @@ -640,7 +644,12 @@ func (rh *RESTHandler) ListJobsHandler(c echo.Context) error { tagsParam := c.QueryParam("tags") var tagsList []string if tagsParam != "" { - tagsList = strings.Split(tagsParam, ",") + for _, t := range strings.Split(tagsParam, ",") { + t = strings.TrimSpace(t) + if t != "" { + tagsList = append(tagsList, t) + } + } } var processIDList []string @@ -694,14 +703,14 @@ func (rh *RESTHandler) ListJobsHandler(c echo.Context) error { links := make([]link, 0) if offset != 0 { lnk := link{ - Href: fmt.Sprintf("/jobs?offset=%v&limit=%v&processID=%v&status=%v&submitter=%v", offset-limit, limit, processIDs, statuses, submitters), + Href: fmt.Sprintf("/jobs?offset=%v&limit=%v&processID=%v&status=%v&submitter=%v&tags=%v", offset-limit, limit, processIDs, statuses, submitters, tagsParam), Title: "prev", } links = append(links, lnk) } if limit == len(result) { lnk := link{ - Href: fmt.Sprintf("/jobs?offset=%v&limit=%v&processID=%v&status=%v&submitter=%v", offset+limit, limit, processIDs, statuses, submitters), + Href: fmt.Sprintf("/jobs?offset=%v&limit=%v&processID=%v&status=%v&submitter=%v&tags=%v", offset+limit, limit, processIDs, statuses, submitters, tagsParam), Title: "next", } links = append(links, lnk) diff --git a/api/jobs/aws_batch_jobs.go b/api/jobs/aws_batch_jobs.go index 60ba6f5..bc1cbb8 100644 --- a/api/jobs/aws_batch_jobs.go +++ b/api/jobs/aws_batch_jobs.go @@ -72,6 +72,10 @@ func (j *AWSBatchJob) ProcessID() string { return j.ProcessName } +func (j *AWSBatchJob) TAGS() []string { + return j.Tags +} + func (j *AWSBatchJob) SUBMITTER() string { return j.Submitter } diff --git a/api/jobs/database_postgres.go b/api/jobs/database_postgres.go index 527e345..f1c4ce1 100644 --- a/api/jobs/database_postgres.go +++ b/api/jobs/database_postgres.go @@ -167,11 +167,11 @@ func (pgDB *PostgresDB) GetJobs(limit, offset int, processIDs, statuses, submitt args = append(args, sb) } } - // Tag filtering: @> means "contains all of" (AND logic) + // Tag filtering: prefix match — each tag term must prefix-match at least one element for _, tag := range tags { - whereClauses = append(whereClauses, fmt.Sprintf("array_to_string(tags, ',') ILIKE $%d", argIndex)) + whereClauses = append(whereClauses, fmt.Sprintf("EXISTS (SELECT 1 FROM unnest(tags) t WHERE t ILIKE $%d)", argIndex)) argIndex++ - args = append(args, "%"+tag+"%") + args = append(args, tag+"%") } if len(whereClauses) > 0 { diff --git a/api/jobs/database_sqlite.go b/api/jobs/database_sqlite.go index da2866c..a171a27 100644 --- a/api/jobs/database_sqlite.go +++ b/api/jobs/database_sqlite.go @@ -184,10 +184,11 @@ func (sqliteDB *SQLiteDB) GetJobs(limit, offset int, processIDs, statuses, submi args = append(args, sb) } } - // Tag filtering: job must contain ALL requested tags (AND logic) + // Tag filtering: prefix match — e.g. "v1" matches "v1", "v1.2", "v1-beta" + // Pattern: wrap column in commas, then match ",%," to anchor at tag boundaries for _, tag := range tags { whereClauses = append(whereClauses, `(',' || LOWER(tags) || ',') LIKE LOWER(?)`) - args = append(args, "%,"+strings.ToLower(tag)+",%") + args = append(args, "%,"+strings.ToLower(tag)+"%,%") } if len(whereClauses) > 0 { diff --git a/api/jobs/docker_jobs.go b/api/jobs/docker_jobs.go index 89b5a4b..286bcf4 100644 --- a/api/jobs/docker_jobs.go +++ b/api/jobs/docker_jobs.go @@ -70,6 +70,10 @@ func (j *DockerJob) SUBMITTER() string { return j.Submitter } +func (j *DockerJob) TAGS() []string { + return j.Tags +} + func (j *DockerJob) CMD() []string { return j.Cmd } diff --git a/api/jobs/jobs.go b/api/jobs/jobs.go index 5ca857c..af2b2e6 100644 --- a/api/jobs/jobs.go +++ b/api/jobs/jobs.go @@ -29,6 +29,7 @@ type Job interface { ProcessID() string ProcessVersionID() string SUBMITTER() string + TAGS() []string // UpdateProcessLogs must provide most upto date process logs // for containerized processes, first fetch the current container logs diff --git a/api/jobs/subprocess_jobs.go b/api/jobs/subprocess_jobs.go index 9096d30..ad570da 100644 --- a/api/jobs/subprocess_jobs.go +++ b/api/jobs/subprocess_jobs.go @@ -68,6 +68,10 @@ func (j *SubprocessJob) SUBMITTER() string { return j.Submitter } +func (j *SubprocessJob) TAGS() []string { + return j.Tags +} + func (j *SubprocessJob) CMD() []string { return j.Cmd } diff --git a/api/public/css/main.css b/api/public/css/main.css index 98af1a4..fd5ef53 100644 --- a/api/public/css/main.css +++ b/api/public/css/main.css @@ -302,4 +302,13 @@ code[class*="language-"] { .legend-available { background-color: var(--table-row-even-bg); +} + +.tag { + display: inline-block; + padding: 0.15rem 0.4rem; + margin: 0.1rem; + font-size: 0.85em; + border: 1px solid var(--table-border-color); + border-radius: 3px; } \ No newline at end of file diff --git a/api/utils/utils.go b/api/utils/utils.go index 6fd99fa..488fc25 100644 --- a/api/utils/utils.go +++ b/api/utils/utils.go @@ -138,18 +138,21 @@ func GetS3LinesData(key string, svc *s3.S3) ([]string, error) { return lines, nil } -var validTagPattern = regexp.MustCompile(`^[A-Za-z0-9._\-]+$`) +var validTagPattern = regexp.MustCompile(`^[A-Za-z0-9._\-:]+$`) -func ValidateTags(tags []string) error { - for _, tag := range tags { +// SanitizeTags trims whitespace from each tag in-place and validates that +// all tags are non-empty and contain only allowed characters (letters, numbers, '.', '-', '_', ':'). +func SanitizeTags(tags []string) error { + for i, tag := range tags { tag = strings.TrimSpace(tag) + tags[i] = tag if tag == "" { return fmt.Errorf("tags cannot contain empty values") } if !validTagPattern.MatchString(tag) { - return fmt.Errorf("tag %q contains unsupported characters. Allowed characters are letters, numbers, '.', '-', and '_'", tag) + return fmt.Errorf("tag %q contains unsupported characters. Allowed characters are letters, numbers, '.', '-', '_', and ':'", tag) } } return nil diff --git a/api/views/jobStatus.html b/api/views/jobStatus.html index 3ea6965..93bd28f 100644 --- a/api/views/jobStatus.html +++ b/api/views/jobStatus.html @@ -13,6 +13,7 @@

Job Status

{{ template "statusTable.html" .}} + {{ template "tagScripts.html" }} diff --git a/api/views/jobs.html b/api/views/jobs.html index 7b4d6e4..c313465 100644 --- a/api/views/jobs.html +++ b/api/views/jobs.html @@ -63,6 +63,7 @@

Jobs List

{{end}} {{end}} + {{ template "tagScripts.html" }} diff --git a/api/views/statusTable.html b/api/views/statusTable.html index 4fb60f4..3bee6a5 100644 --- a/api/views/statusTable.html +++ b/api/views/statusTable.html @@ -24,6 +24,10 @@ {{.Status}} + + Tags + {{range .Tags}}{{.}} {{end}} + Last Updated {{.LastUpdate.Format "2006-01-02 15:04:05 MST"}} diff --git a/api/views/tagScripts.html b/api/views/tagScripts.html new file mode 100644 index 0000000..380a5e0 --- /dev/null +++ b/api/views/tagScripts.html @@ -0,0 +1,14 @@ + diff --git a/tests/e2e/tests.postman_collection.json b/tests/e2e/tests.postman_collection.json index b1d09bd..44ee6af 100644 --- a/tests/e2e/tests.postman_collection.json +++ b/tests/e2e/tests.postman_collection.json @@ -3122,7 +3122,7 @@ "response": [] }, { - "name": "fetch-with-tag-substring-xas", + "name": "fetch-with-tag-non-prefix-xas", "event": [ { "listen": "test", @@ -3130,11 +3130,8 @@ "type": "text/javascript", "exec": [ "const jobsData = pm.response.json().jobs;", - "pm.test(\"Searching tags by substring xas returns texas-tagged jobs\", function () {", - " pm.expect(jobsData.length).to.be.above(0);", - " jobsData.forEach(job => {", - " pm.expect(job.tags.some(tag => tag.includes(\"xas\"))).to.eql(true);", - " });", + "pm.test(\"Searching tags by non-prefix 'xas' returns no jobs\", function () {", + " pm.expect(jobsData.length).to.eql(0);", "});" ] }