From d6214682f4e9b7ac95d6d4ae3a1de6349fc9acfd Mon Sep 17 00:00:00 2001 From: Eunice Chan Date: Thu, 1 Jun 2023 10:37:28 -0700 Subject: [PATCH 1/6] Init impl of routes --- .../handler/v2/node_check_results_get.go | 141 ++++++++++++++++++ .../handler/v2/node_metric_results_get.go | 139 +++++++++++++++++ src/golang/cmd/server/routes/routes.go | 4 +- src/golang/cmd/server/server/handlers.go | 14 ++ src/golang/lib/repos/operator.go | 6 + src/golang/lib/repos/operator_result.go | 3 + src/golang/lib/repos/sqlite/operator.go | 26 ++++ src/ui/common/src/handlers/AqueductApi.ts | 16 ++ .../src/handlers/v2/NodeCheckResultsGet.ts | 24 +++ .../src/handlers/v2/NodeMetricResultsGet.ts | 24 +++ 10 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 src/golang/cmd/server/handler/v2/node_check_results_get.go create mode 100644 src/golang/cmd/server/handler/v2/node_metric_results_get.go create mode 100644 src/ui/common/src/handlers/v2/NodeCheckResultsGet.ts create mode 100644 src/ui/common/src/handlers/v2/NodeMetricResultsGet.ts diff --git a/src/golang/cmd/server/handler/v2/node_check_results_get.go b/src/golang/cmd/server/handler/v2/node_check_results_get.go new file mode 100644 index 0000000000..f2dd30e0d5 --- /dev/null +++ b/src/golang/cmd/server/handler/v2/node_check_results_get.go @@ -0,0 +1,141 @@ +package v2 + +import ( + "context" + "fmt" + "net/http" + + "github.com/aqueducthq/aqueduct/cmd/server/handler" + "github.com/aqueducthq/aqueduct/lib/database" + "github.com/aqueducthq/aqueduct/lib/models" + "github.com/aqueducthq/aqueduct/lib/models/shared" + "github.com/aqueducthq/aqueduct/lib/repos" + "github.com/aqueducthq/aqueduct/lib/response" + "github.com/aqueducthq/aqueduct/lib/storage" + "github.com/dropbox/godropbox/errors" + "github.com/google/uuid" +) + +// This file should map directly to +// src/ui/common/src/handlers/v2/NodeCheckResultsGet.tsx +// +// Returns all downstream artifact results +// Route: /api/v2/workflow/{workflowID}/dag/{dagID}/node/check/{nodeID}/results +// Method: GET +// Params: +// `workflowID`: ID for `workflow` object +// `dagID`: ID for `workflow_dag` object +// `nodeID`: ID for operator object +// Request: +// Headers: +// `api-key`: user's API Key +// Response: +// Body: +// `[]response.OperatorWithArtifactNodeResult` + +type NodeCheckResultsGetHandler struct { + nodeGetHandler + handler.GetHandler + + Database database.Database + + WorkflowRepo repos.Workflow + DAGRepo repos.DAG + OperatorRepo repos.Operator + OperatorResultRepo repos.OperatorResult + ArtifactRepo repos.Artifact + ArtifactResultRepo repos.ArtifactResult +} + +func (*NodeCheckResultsGetHandler) Name() string { + return "NodeCheckResultsGet" +} + +func (h *NodeCheckResultsGetHandler) Prepare(r *http.Request) (interface{}, int, error) { + return h.nodeGetHandler.Prepare(r) +} + +func (h *NodeCheckResultsGetHandler) Perform(ctx context.Context, interfaceArgs interface{}) (interface{}, int, error) { + args := interfaceArgs.(*nodeGetArgs) + + artfID := args.nodeID + wfID := args.workflowID + + emptyResponse := []response.OperatorWithArtifactNodeResult{} + + dbOperatorWithArtifactNode, err := h.OperatorRepo.GetOperatorWithArtifactNodeByArtifactId(ctx, artfID, h.Database) + if err != nil { + return nil, http.StatusInternalServerError, errors.Wrap(err, "Unexpected error reading check node.") + } + + results, err := h.OperatorResultRepo.GetOperatorWithArtifactNodeByOperatorNameAndWorkflow(ctx, dbOperatorWithArtifactNode.Name, wfID, h.Database) + if err != nil { + return emptyResponse, http.StatusInternalServerError, errors.Wrap(err, "Unable to retrieve check results.") + } + + if len(results) == 0 { + return emptyResponse, http.StatusOK, nil + } + + resultArtifactIds := make([]uuid.UUID, 0, len(results)) + for _, result := range results { + resultArtifactIds = append(resultArtifactIds, result.ArtifactID) + } + + artfResultToDAG, err := h.DAGRepo.GetByArtifactResultBatch(ctx, resultArtifactIds, h.Database) + if err != nil { + return emptyResponse, http.StatusInternalServerError, errors.Wrap(err, "Unable to retrieve workflow dags.") + } + + // maps from db dag Ids + dbDagByDagId := make(map[uuid.UUID]models.DAG, len(artfResultToDAG)) + nodeResultByDagId := make(map[uuid.UUID][]models.OperatorWithArtifactResult, len(artfResultToDAG)) + for _, artfResult := range results { + if dbDag, ok := artfResultToDAG[artfResult.ID]; ok { + if _, okDagsMap := dbDagByDagId[dbDag.ID]; !okDagsMap { + dbDagByDagId[dbDag.ID] = dbDag + } + + nodeResultByDagId[dbDag.ID] = append(nodeResultByDagId[dbDag.ID], artfResult) + } else { + return emptyResponse, http.StatusInternalServerError, errors.Newf("Error retrieving dag associated with artifact result %s", artfResult.ID) + } + } + + responses := make([]response.OperatorWithArtifactResult, 0, len(results)) + for dbDagId, artfResults := range nodeResultByDagId { + if dag, ok := dbDagByDagId[dbDagId]; ok { + storageObj := storage.NewStorage(&dag.StorageConfig) + if err != nil { + return emptyResponse, http.StatusInternalServerError, errors.New("Error retrieving artifact contents.") + } + + for _, artfResult := range artfResults { + var contentPtr *string = nil + if artf.Type.IsCompact() && + !artfResult.ExecState.IsNull && + (artfResult.ExecState.ExecutionState.Status == shared.FailedExecutionStatus || + artfResult.ExecState.ExecutionState.Status == shared.SucceededExecutionStatus) { + exists := storageObj.Exists(ctx, artfResult.ContentPath) + if exists { + contentBytes, err := storageObj.Get(ctx, artfResult.ContentPath) + if err != nil { + return emptyResponse, http.StatusInternalServerError, errors.Wrap(err, fmt.Sprintf("Error retrieving artifact content for result %s", artfResult.ID)) + } + + contentStr := string(contentBytes) + contentPtr = &contentStr + } + } + + responses = append(responses, *response.NewOperatorWithArtifactNodeResultFromDBObject( + &artfResult, contentPtr, + )) + } + } else { + return emptyResponse, http.StatusInternalServerError, errors.Newf("Error retrieving dag %s", dbDagId) + } + } + + return responses, http.StatusOK, nil +} diff --git a/src/golang/cmd/server/handler/v2/node_metric_results_get.go b/src/golang/cmd/server/handler/v2/node_metric_results_get.go new file mode 100644 index 0000000000..94b80294cd --- /dev/null +++ b/src/golang/cmd/server/handler/v2/node_metric_results_get.go @@ -0,0 +1,139 @@ +package v2 + +import ( + "context" + "fmt" + "net/http" + + "github.com/aqueducthq/aqueduct/cmd/server/handler" + "github.com/aqueducthq/aqueduct/lib/database" + "github.com/aqueducthq/aqueduct/lib/models" + "github.com/aqueducthq/aqueduct/lib/models/shared" + "github.com/aqueducthq/aqueduct/lib/repos" + "github.com/aqueducthq/aqueduct/lib/response" + "github.com/aqueducthq/aqueduct/lib/storage" + "github.com/dropbox/godropbox/errors" + "github.com/google/uuid" +) + +// This file should map directly to +// src/ui/common/src/handlers/v2/NodeMetricResultsGet.tsx +// +// Returns all downstream artifact results +// Route: /api/v2/workflow/{workflowID}/dag/{dagID}/node/metric/{nodeID}/results +// Method: GET +// Params: +// `workflowID`: ID for `workflow` object +// `dagID`: ID for `workflow_dag` object +// `nodeID`: ID for operator object +// Request: +// Headers: +// `api-key`: user's API Key +// Response: +// Body: +// `[]response.OperatorWithArtifactNodeResult` + +type NodeMetricResultsGetHandler struct { + nodeGetHandler + handler.GetHandler + + Database database.Database + + WorkflowRepo repos.Workflow + DAGRepo repos.DAG + OperatorRepo repos.Operator + OperatorResultRepo repos.OperatorResult +} + +func (*NodeMetricResultsGetHandler) Name() string { + return "NodeMetricResultsGet" +} + +func (h *NodeMetricResultsGetHandler) Prepare(r *http.Request) (interface{}, int, error) { + return h.nodeGetHandler.Prepare(r) +} + +func (h *NodeMetricResultsGetHandler) Perform(ctx context.Context, interfaceArgs interface{}) (interface{}, int, error) { + args := interfaceArgs.(*nodeGetArgs) + + artfID := args.nodeID + wfID := args.workflowID + + emptyResponse := []response.OperatorWithArtifactNodeResult{} + + dbOperatorWithArtifactNode, err := h.OperatorRepo.GetOperatorWithArtifactNodeByArtifactId(ctx, artfID, h.Database) + if err != nil { + return nil, http.StatusInternalServerError, errors.Wrap(err, "Unexpected error reading metric node.") + } + + results, err := h.OperatorResultRepo.GetOperatorWithArtifactNodeByOperatorNameAndWorkflow(ctx, dbOperatorWithArtifactNode.Name, wfID, h.Database) + if err != nil { + return emptyResponse, http.StatusInternalServerError, errors.Wrap(err, "Unable to retrieve metric results.") + } + + if len(results) == 0 { + return emptyResponse, http.StatusOK, nil + } + + resultArtifactIds := make([]uuid.UUID, 0, len(results)) + for _, result := range results { + resultArtifactIds = append(resultArtifactIds, result.ArtifactID) + } + + artfResultToDAG, err := h.DAGRepo.GetByArtifactResultBatch(ctx, resultArtifactIds, h.Database) + if err != nil { + return emptyResponse, http.StatusInternalServerError, errors.Wrap(err, "Unable to retrieve workflow dags.") + } + + // maps from db dag Ids + dbDagByDagId := make(map[uuid.UUID]models.DAG, len(artfResultToDAG)) + nodeResultByDagId := make(map[uuid.UUID][]models.OperatorWithArtifactResult, len(artfResultToDAG)) + for _, artfResult := range results { + if dbDag, ok := artfResultToDAG[artfResult.ID]; ok { + if _, okDagsMap := dbDagByDagId[dbDag.ID]; !okDagsMap { + dbDagByDagId[dbDag.ID] = dbDag + } + + nodeResultByDagId[dbDag.ID] = append(nodeResultByDagId[dbDag.ID], artfResult) + } else { + return emptyResponse, http.StatusInternalServerError, errors.Newf("Error retrieving dag associated with artifact result %s", artfResult.ID) + } + } + + responses := make([]response.OperatorWithArtifactResult, 0, len(results)) + for dbDagId, artfResults := range nodeResultByDagId { + if dag, ok := dbDagByDagId[dbDagId]; ok { + storageObj := storage.NewStorage(&dag.StorageConfig) + if err != nil { + return emptyResponse, http.StatusInternalServerError, errors.New("Error retrieving artifact contents.") + } + + for _, artfResult := range artfResults { + var contentPtr *string = nil + if artf.Type.IsCompact() && + !artfResult.ExecState.IsNull && + (artfResult.ExecState.ExecutionState.Status == shared.FailedExecutionStatus || + artfResult.ExecState.ExecutionState.Status == shared.SucceededExecutionStatus) { + exists := storageObj.Exists(ctx, artfResult.ContentPath) + if exists { + contentBytes, err := storageObj.Get(ctx, artfResult.ContentPath) + if err != nil { + return emptyResponse, http.StatusInternalServerError, errors.Wrap(err, fmt.Sprintf("Error retrieving artifact content for result %s", artfResult.ID)) + } + + contentStr := string(contentBytes) + contentPtr = &contentStr + } + } + + responses = append(responses, *response.NewOperatorWithArtifactNodeResultFromDBObject( + &artfResult, contentPtr, + )) + } + } else { + return emptyResponse, http.StatusInternalServerError, errors.Newf("Error retrieving dag %s", dbDagId) + } + } + + return responses, http.StatusOK, nil +} diff --git a/src/golang/cmd/server/routes/routes.go b/src/golang/cmd/server/routes/routes.go index e358bfd602..ba2d5fb2c0 100644 --- a/src/golang/cmd/server/routes/routes.go +++ b/src/golang/cmd/server/routes/routes.go @@ -17,11 +17,13 @@ const ( NodesRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/nodes" NodeArtifactRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/artifact/{nodeID}" NodeArtifactResultContentRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/artifact/{nodeID}/result/{nodeResultID}/content" - NodeArtifactResultsRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/artifact/{nodeID}/results" + NodeArtifactResultsRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/artifact/{nodeID}/results" NodeMetricRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/metric/{nodeID}" NodeMetricResultContentRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/metric/{nodeID}/result/{nodeResultID}/content" + NodeMetricResultsRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/metric/{nodeID}/results" NodeCheckRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/check/{nodeID}" NodeCheckResultContentRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/check/{nodeID}/result/{nodeResultID}/content" + NodeCheckResultsRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/check/{nodeID}/results" NodeOperatorRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/operator/{nodeID}" NodeDagOperatorsRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/operators" NodeOperatorContentRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/operator/{nodeID}/content" diff --git a/src/golang/cmd/server/server/handlers.go b/src/golang/cmd/server/server/handlers.go index 28be1b3617..748013415c 100644 --- a/src/golang/cmd/server/server/handlers.go +++ b/src/golang/cmd/server/server/handlers.go @@ -84,6 +84,13 @@ func (s *AqServer) Handlers() map[string]handler.Handler { ArtifactRepo: s.ArtifactRepo, ArtifactResultRepo: s.ArtifactResultRepo, }, + routes.NodeMetricResultsRoute: &v2.NodeMetricResultsGetHandler{ + Database: s.Database, + WorkflowRepo: s.WorkflowRepo, + DAGRepo: s.DAGRepo, + OperatorRepo: s.OperatorRepo, + OperatorResultRepo: s.OperatorResultRepo, + }, routes.NodeCheckRoute: &v2.NodeCheckGetHandler{ Database: s.Database, WorkflowRepo: s.WorkflowRepo, @@ -103,6 +110,13 @@ func (s *AqServer) Handlers() map[string]handler.Handler { DAGRepo: s.DAGRepo, OperatorRepo: s.OperatorRepo, }, + routes.NodeCheckResultsRoute: &v2.NodeCheckResultsGetHandler{ + Database: s.Database, + WorkflowRepo: s.WorkflowRepo, + DAGRepo: s.DAGRepo, + OperatorRepo: s.OperatorRepo, + OperatorResultRepo: s.OperatorResultRepo, + }, routes.NodeOperatorRoute: &v2.NodeOperatorGetHandler{ Database: s.Database, WorkflowRepo: s.WorkflowRepo, diff --git a/src/golang/lib/repos/operator.go b/src/golang/lib/repos/operator.go index 1d2891649b..546f5a60a8 100644 --- a/src/golang/lib/repos/operator.go +++ b/src/golang/lib/repos/operator.go @@ -36,6 +36,12 @@ type operatorReader interface { // GetOperatorWithArtifactNodeBatch returns the OperatorWithArtifactNode views given the operator IDs. GetOperatorWithArtifactNodeBatch(ctx context.Context, IDs []uuid.UUID, DB database.Database) ([]views.OperatorWithArtifactNode, error) + // GetOperatorWithArtifactByArtifactIdNode returns the OperatorWithArtifactNode view given the artifact ID. + GetOperatorWithArtifactByArtifactIdNode(ctx context.Context, artifactID uuid.UUID, DB database.Database) (*views.OperatorWithArtifactNode, error) + + // GetOperatorWithArtifactByArtifactIdNodeBatch returns the OperatorWithArtifactNode views given the artifact IDs. + GetOperatorWithArtifactByArtifactIdNodeBatch(ctx context.Context, artifactIDs []uuid.UUID, DB database.Database) ([]views.OperatorWithArtifactNode, error) + // GetBatch returns the Operators with IDs. GetBatch(ctx context.Context, IDs []uuid.UUID, DB database.Database) ([]models.Operator, error) diff --git a/src/golang/lib/repos/operator_result.go b/src/golang/lib/repos/operator_result.go index 1b06a25980..438f7bcd86 100644 --- a/src/golang/lib/repos/operator_result.go +++ b/src/golang/lib/repos/operator_result.go @@ -30,6 +30,9 @@ type operatorResultReader interface { // GetByDAGResultBatch returns all OperatorResults for the DAGResults specified. GetByDAGResultBatch(ctx context.Context, dagResultIDs []uuid.UUID, DB database.Database) ([]models.OperatorResult, error) + // GetOperatorWithArtifactNodeByOperatorNameAndWorkflow returns the OperatorWithArtifactNode for the Workflow and Operator specified. + GetOperatorWithArtifactNodeByOperatorNameAndWorkflow(ctx context.Context, dagResultID, operatorName string, workflowID uuid.UUID, DB database.Database) (*models.OperatorWithArtifactNodeResult, error) + // GetCheckStatusByArtifactBatch returns an OperatorResultStatus for all OperatorResults // associated with a Check Operator where the Operator has incoming DAGEdge // from an Artifact in artifactIDs. diff --git a/src/golang/lib/repos/sqlite/operator.go b/src/golang/lib/repos/sqlite/operator.go index ba102aebca..7ad571dd24 100644 --- a/src/golang/lib/repos/sqlite/operator.go +++ b/src/golang/lib/repos/sqlite/operator.go @@ -200,6 +200,32 @@ func (*operatorReader) GetOperatorWithArtifactNodeBatch(ctx context.Context, IDs return getOperatorWithArtifactNodes(ctx, DB, query, args...) } +func (r *operatorReader) GetOperatorWithArtifactNodeByArtifactId(ctx context.Context, artifactID uuid.UUID, DB database.Database) (*views.OperatorWithArtifactNode, error) { + nodes, err := r.GetOperatorWithArtifactNodeBatch(ctx, []uuid.UUID{artifactID}, DB) + if err != nil { + return nil, err + } + return &nodes[0], nil +} + +func (*operatorReader) GetOperatorWithArtifactNodeByArtifactIdBatch(ctx context.Context, artifactIDs []uuid.UUID, DB database.Database) ([]views.OperatorWithArtifactNode, error) { + if len(IDs) == 0 { + return nil, errors.New("Provided empty IDs list.") + } + + query := fmt.Sprintf( + "WITH %s AS (%s) SELECT %s FROM %s WHERE %s IN (%s)", + views.OperatorWithArtifactNodeView, + mergedNodeViewSubQuery, + views.OperatorWithArtifactNodeCols(), + views.OperatorWithArtifactNodeView, + views.OperatorWithArtifactNodeArtifactID, + stmt_preparers.GenerateArgsList(len(artifactIDs), 1), + ) + args := stmt_preparers.CastIdsListToInterfaceList(IDs) + return getOperatorWithArtifactNodes(ctx, DB, query, args...) +} + func (*operatorReader) GetBatch(ctx context.Context, IDs []uuid.UUID, DB database.Database) ([]models.Operator, error) { if len(IDs) == 0 { return nil, errors.New("Provided empty IDs list.") diff --git a/src/ui/common/src/handlers/AqueductApi.ts b/src/ui/common/src/handlers/AqueductApi.ts index 62b38c48dc..60e98022ad 100644 --- a/src/ui/common/src/handlers/AqueductApi.ts +++ b/src/ui/common/src/handlers/AqueductApi.ts @@ -213,6 +213,20 @@ export const aqueductApi = createApi({ query: (req) => nodeArtifactResultsGetQuery(req), transformErrorResponse, }), + nodeMetricResultsGet: builder.query< + NodeMetricResultsGetResponse, + NodeMetricResultsGetRequest + >({ + query: (req) => nodeMetricResultsGetQuery(req), + transformErrorResponse, + }), + nodeCheckResultsGet: builder.query< + NodeCheckResultsGetResponse, + NodeCheckResultsGetRequest + >({ + query: (req) => nodeCheckResultsGetQuery(req), + transformErrorResponse, + }), nodeOperatorGet: builder.query< NodeOperatorGetResponse, NodeOperatorGetRequest @@ -320,6 +334,8 @@ export const { useNodeArtifactGetQuery, useNodeArtifactResultContentGetQuery, useNodeArtifactResultsGetQuery, + useNodeMetricResultsGetQuery, + useNodeCheckResultsGetQuery, useNodeOperatorGetQuery, useNodeOperatorContentGetQuery, useNodeMetricGetQuery, diff --git a/src/ui/common/src/handlers/v2/NodeCheckResultsGet.ts b/src/ui/common/src/handlers/v2/NodeCheckResultsGet.ts new file mode 100644 index 0000000000..254eca3eee --- /dev/null +++ b/src/ui/common/src/handlers/v2/NodeCheckResultsGet.ts @@ -0,0 +1,24 @@ +// This file should map exactly to +// src/golang/cmd/server/handler/v2/node_check_results_get.go + +import { APIKeyParameter } from '../parameters/Header'; +import { + DagIdParameter, + NodeIdParameter, + WorkflowIdParameter, +} from '../parameters/Path'; +import { OperatorWithArtifactNodeResultResponse } from '../responses/node'; + +export type NodeCheckResultsGetRequest = APIKeyParameter & + DagIdParameter & + NodeIdParameter & + WorkflowIdParameter; + +export type NodeCheckResultsGetResponse = OperatorWithArtifactNodeResultResponse[]; + +export const nodeCheckResultsGetQuery = ( + req: NodeCheckResultsGetRequest +) => ({ + url: `workflow/${req.workflowId}/dag/${req.dagId}/node/check/${req.nodeId}/results`, + headers: { 'api-key': req.apiKey }, +}); diff --git a/src/ui/common/src/handlers/v2/NodeMetricResultsGet.ts b/src/ui/common/src/handlers/v2/NodeMetricResultsGet.ts new file mode 100644 index 0000000000..16a7605ff9 --- /dev/null +++ b/src/ui/common/src/handlers/v2/NodeMetricResultsGet.ts @@ -0,0 +1,24 @@ +// This file should map exactly to +// src/golang/cmd/server/handler/v2/node_metric_results_get.go + +import { APIKeyParameter } from '../parameters/Header'; +import { + DagIdParameter, + NodeIdParameter, + WorkflowIdParameter, +} from '../parameters/Path'; +import { OperatorWithArtifactNodeResultResponse } from '../responses/node'; + +export type NodeMetricResultsGetRequest = APIKeyParameter & + DagIdParameter & + NodeIdParameter & + WorkflowIdParameter; + +export type NodeMetricResultsGetResponse = OperatorWithArtifactNodeResultResponse[]; + +export const nodeMetricResultsGetQuery = ( + req: NodeMetricResultsGetResponse +) => ({ + url: `workflow/${req.workflowId}/dag/${req.dagId}/node/metric/${req.nodeId}/results`, + headers: { 'api-key': req.apiKey }, +}); From 17dedcb3478d1b0b0c094806189746f9cbe1585c Mon Sep 17 00:00:00 2001 From: Eunice Chan Date: Fri, 2 Jun 2023 11:14:46 -0700 Subject: [PATCH 2/6] Routes done --- .../handler/v2/node_check_results_get.go | 34 +++++----- .../handler/v2/node_metric_results_get.go | 34 +++++----- .../lib/models/views/merged_node_result.go | 58 ----------------- ...node.go => operator_with_artifact_node.go} | 2 +- .../operator_with_artifact_node_result.go | 64 +++++++++++++++++++ src/golang/lib/repos/operator_result.go | 4 +- src/golang/lib/repos/sqlite/operator.go | 10 +-- .../lib/repos/sqlite/operator_result.go | 38 +++++++++++ src/golang/lib/response/node.go | 46 ++++++------- 9 files changed, 168 insertions(+), 122 deletions(-) delete mode 100644 src/golang/lib/models/views/merged_node_result.go rename src/golang/lib/models/views/{merged_node.go => operator_with_artifact_node.go} (96%) create mode 100644 src/golang/lib/models/views/operator_with_artifact_node_result.go diff --git a/src/golang/cmd/server/handler/v2/node_check_results_get.go b/src/golang/cmd/server/handler/v2/node_check_results_get.go index f2dd30e0d5..ae78ad5521 100644 --- a/src/golang/cmd/server/handler/v2/node_check_results_get.go +++ b/src/golang/cmd/server/handler/v2/node_check_results_get.go @@ -13,6 +13,7 @@ import ( "github.com/aqueducthq/aqueduct/lib/response" "github.com/aqueducthq/aqueduct/lib/storage" "github.com/dropbox/godropbox/errors" + "github.com/aqueducthq/aqueduct/lib/models/views" "github.com/google/uuid" ) @@ -31,7 +32,7 @@ import ( // `api-key`: user's API Key // Response: // Body: -// `[]response.OperatorWithArtifactNodeResult` +// `[]response.OperatorWithArtifactResultNode` type NodeCheckResultsGetHandler struct { nodeGetHandler @@ -61,14 +62,14 @@ func (h *NodeCheckResultsGetHandler) Perform(ctx context.Context, interfaceArgs artfID := args.nodeID wfID := args.workflowID - emptyResponse := []response.OperatorWithArtifactNodeResult{} + emptyResponse := []response.OperatorWithArtifactResultNode{} - dbOperatorWithArtifactNode, err := h.OperatorRepo.GetOperatorWithArtifactNodeByArtifactId(ctx, artfID, h.Database) + dbOperatorWithArtifactNode, err := h.OperatorRepo.GetOperatorWithArtifactByArtifactIdNode(ctx, artfID, h.Database) if err != nil { return nil, http.StatusInternalServerError, errors.Wrap(err, "Unexpected error reading check node.") } - results, err := h.OperatorResultRepo.GetOperatorWithArtifactNodeByOperatorNameAndWorkflow(ctx, dbOperatorWithArtifactNode.Name, wfID, h.Database) + results, err := h.OperatorResultRepo.GetOperatorWithArtifactResultNodesByOperatorNameAndWorkflow(ctx, dbOperatorWithArtifactNode.Name, wfID, h.Database) if err != nil { return emptyResponse, http.StatusInternalServerError, errors.Wrap(err, "Unable to retrieve check results.") } @@ -89,7 +90,7 @@ func (h *NodeCheckResultsGetHandler) Perform(ctx context.Context, interfaceArgs // maps from db dag Ids dbDagByDagId := make(map[uuid.UUID]models.DAG, len(artfResultToDAG)) - nodeResultByDagId := make(map[uuid.UUID][]models.OperatorWithArtifactResult, len(artfResultToDAG)) + nodeResultByDagId := make(map[uuid.UUID][]views.OperatorWithArtifactResultNode, len(artfResultToDAG)) for _, artfResult := range results { if dbDag, ok := artfResultToDAG[artfResult.ID]; ok { if _, okDagsMap := dbDagByDagId[dbDag.ID]; !okDagsMap { @@ -102,25 +103,24 @@ func (h *NodeCheckResultsGetHandler) Perform(ctx context.Context, interfaceArgs } } - responses := make([]response.OperatorWithArtifactResult, 0, len(results)) - for dbDagId, artfResults := range nodeResultByDagId { + responses := make([]response.OperatorWithArtifactResultNode, 0, len(results)) + for dbDagId, nodeResults := range nodeResultByDagId { if dag, ok := dbDagByDagId[dbDagId]; ok { storageObj := storage.NewStorage(&dag.StorageConfig) if err != nil { return emptyResponse, http.StatusInternalServerError, errors.New("Error retrieving artifact contents.") } - for _, artfResult := range artfResults { + for _, nodeResult := range nodeResults { var contentPtr *string = nil - if artf.Type.IsCompact() && - !artfResult.ExecState.IsNull && - (artfResult.ExecState.ExecutionState.Status == shared.FailedExecutionStatus || - artfResult.ExecState.ExecutionState.Status == shared.SucceededExecutionStatus) { - exists := storageObj.Exists(ctx, artfResult.ContentPath) + if !nodeResult.ArtifactResultExecState.IsNull && + (nodeResult.ArtifactResultExecState.ExecutionState.Status == shared.FailedExecutionStatus || + nodeResult.ArtifactResultExecState.ExecutionState.Status == shared.SucceededExecutionStatus) { + exists := storageObj.Exists(ctx, nodeResult.ContentPath) if exists { - contentBytes, err := storageObj.Get(ctx, artfResult.ContentPath) + contentBytes, err := storageObj.Get(ctx, nodeResult.ContentPath) if err != nil { - return emptyResponse, http.StatusInternalServerError, errors.Wrap(err, fmt.Sprintf("Error retrieving artifact content for result %s", artfResult.ID)) + return emptyResponse, http.StatusInternalServerError, errors.Wrap(err, fmt.Sprintf("Error retrieving artifact content for result %s", nodeResult.ArtifactID)) } contentStr := string(contentBytes) @@ -128,8 +128,8 @@ func (h *NodeCheckResultsGetHandler) Perform(ctx context.Context, interfaceArgs } } - responses = append(responses, *response.NewOperatorWithArtifactNodeResultFromDBObject( - &artfResult, contentPtr, + responses = append(responses, *response.NewOperatorWithArtifactResultNodeFromDBObject( + &nodeResult, contentPtr, )) } } else { diff --git a/src/golang/cmd/server/handler/v2/node_metric_results_get.go b/src/golang/cmd/server/handler/v2/node_metric_results_get.go index 94b80294cd..38006779ef 100644 --- a/src/golang/cmd/server/handler/v2/node_metric_results_get.go +++ b/src/golang/cmd/server/handler/v2/node_metric_results_get.go @@ -13,6 +13,7 @@ import ( "github.com/aqueducthq/aqueduct/lib/response" "github.com/aqueducthq/aqueduct/lib/storage" "github.com/dropbox/godropbox/errors" + "github.com/aqueducthq/aqueduct/lib/models/views" "github.com/google/uuid" ) @@ -31,7 +32,7 @@ import ( // `api-key`: user's API Key // Response: // Body: -// `[]response.OperatorWithArtifactNodeResult` +// `[]response.OperatorWithArtifactResultNode` type NodeMetricResultsGetHandler struct { nodeGetHandler @@ -59,14 +60,14 @@ func (h *NodeMetricResultsGetHandler) Perform(ctx context.Context, interfaceArgs artfID := args.nodeID wfID := args.workflowID - emptyResponse := []response.OperatorWithArtifactNodeResult{} + emptyResponse := []response.OperatorWithArtifactResultNode{} - dbOperatorWithArtifactNode, err := h.OperatorRepo.GetOperatorWithArtifactNodeByArtifactId(ctx, artfID, h.Database) + dbOperatorWithArtifactNode, err := h.OperatorRepo.GetOperatorWithArtifactByArtifactIdNode(ctx, artfID, h.Database) if err != nil { return nil, http.StatusInternalServerError, errors.Wrap(err, "Unexpected error reading metric node.") } - results, err := h.OperatorResultRepo.GetOperatorWithArtifactNodeByOperatorNameAndWorkflow(ctx, dbOperatorWithArtifactNode.Name, wfID, h.Database) + results, err := h.OperatorResultRepo.GetOperatorWithArtifactResultNodesByOperatorNameAndWorkflow(ctx, dbOperatorWithArtifactNode.Name, wfID, h.Database) if err != nil { return emptyResponse, http.StatusInternalServerError, errors.Wrap(err, "Unable to retrieve metric results.") } @@ -87,7 +88,7 @@ func (h *NodeMetricResultsGetHandler) Perform(ctx context.Context, interfaceArgs // maps from db dag Ids dbDagByDagId := make(map[uuid.UUID]models.DAG, len(artfResultToDAG)) - nodeResultByDagId := make(map[uuid.UUID][]models.OperatorWithArtifactResult, len(artfResultToDAG)) + nodeResultByDagId := make(map[uuid.UUID][]views.OperatorWithArtifactResultNode, len(artfResultToDAG)) for _, artfResult := range results { if dbDag, ok := artfResultToDAG[artfResult.ID]; ok { if _, okDagsMap := dbDagByDagId[dbDag.ID]; !okDagsMap { @@ -100,25 +101,24 @@ func (h *NodeMetricResultsGetHandler) Perform(ctx context.Context, interfaceArgs } } - responses := make([]response.OperatorWithArtifactResult, 0, len(results)) - for dbDagId, artfResults := range nodeResultByDagId { + responses := make([]response.OperatorWithArtifactResultNode, 0, len(results)) + for dbDagId, nodeResults := range nodeResultByDagId { if dag, ok := dbDagByDagId[dbDagId]; ok { storageObj := storage.NewStorage(&dag.StorageConfig) if err != nil { return emptyResponse, http.StatusInternalServerError, errors.New("Error retrieving artifact contents.") } - for _, artfResult := range artfResults { + for _, nodeResult := range nodeResults { var contentPtr *string = nil - if artf.Type.IsCompact() && - !artfResult.ExecState.IsNull && - (artfResult.ExecState.ExecutionState.Status == shared.FailedExecutionStatus || - artfResult.ExecState.ExecutionState.Status == shared.SucceededExecutionStatus) { - exists := storageObj.Exists(ctx, artfResult.ContentPath) + if !nodeResult.ArtifactResultExecState.IsNull && + (nodeResult.ArtifactResultExecState.ExecutionState.Status == shared.FailedExecutionStatus || + nodeResult.ArtifactResultExecState.ExecutionState.Status == shared.SucceededExecutionStatus) { + exists := storageObj.Exists(ctx, nodeResult.ContentPath) if exists { - contentBytes, err := storageObj.Get(ctx, artfResult.ContentPath) + contentBytes, err := storageObj.Get(ctx, nodeResult.ContentPath) if err != nil { - return emptyResponse, http.StatusInternalServerError, errors.Wrap(err, fmt.Sprintf("Error retrieving artifact content for result %s", artfResult.ID)) + return emptyResponse, http.StatusInternalServerError, errors.Wrap(err, fmt.Sprintf("Error retrieving artifact content for result %s", nodeResult.ArtifactID)) } contentStr := string(contentBytes) @@ -126,8 +126,8 @@ func (h *NodeMetricResultsGetHandler) Perform(ctx context.Context, interfaceArgs } } - responses = append(responses, *response.NewOperatorWithArtifactNodeResultFromDBObject( - &artfResult, contentPtr, + responses = append(responses, *response.NewOperatorWithArtifactResultNodeFromDBObject( + &nodeResult, contentPtr, )) } } else { diff --git a/src/golang/lib/models/views/merged_node_result.go b/src/golang/lib/models/views/merged_node_result.go deleted file mode 100644 index b3e8f71de0..0000000000 --- a/src/golang/lib/models/views/merged_node_result.go +++ /dev/null @@ -1,58 +0,0 @@ -package views - -import ( - "fmt" - "strings" - - "github.com/aqueducthq/aqueduct/lib/models/shared" - "github.com/google/uuid" -) - -const ( - OperatorWithArtifactNodeResultTable = "merged_node_result" - - // OperatorWithArtifactNodeResult table column names - OperatorWithArtifactNodeResultID = "id" - OperatorWithArtifactNodeResultOperatorExecState = "operator_exec_state" - OperatorWithArtifactNodeResultArtifactID = "artifact_id" - OperatorWithArtifactNodeResultMetadata = "metadata" - OperatorWithArtifactNodeResultContentPath = "content_path" - OperatorWithArtifactNodeResultArtifactExecState = "artifact_exec_state" -) - -// An OperatorWithArtifactNodeResult maps to the merged_node_result table. -type OperatorWithArtifactNodeResult struct { - ID uuid.UUID `db:"id" json:"id"` - OperatorExecState shared.NullExecutionState `db:"operator_exec_state" json:"operator_exec_state"` - ArtifactID uuid.UUID `db:"artifact_id" json:"artifact_id"` - Metadata shared.NullArtifactResultMetadata `db:"metadata" json:"metadata"` - ContentPath string `db:"content_path" json:"content_path"` - ArtifactExecState shared.NullExecutionState `db:"artifact_exec_state" json:"artifact_exec_state"` -} - -// OperatorWithArtifactNodeResultCols returns a comma-separated string of all OperatorWithArtifactNodeResult columns. -func OperatorWithArtifactNodeResultCols() string { - return strings.Join(allOperatorWithArtifactNodeResultCols(), ",") -} - -// OperatorWithArtifactNodeResultColsWithPrefix returns a comma-separated string of all -// OperatorWithArtifactNodeResult columns prefixed by the table name. -func OperatorWithArtifactNodeResultColsWithPrefix() string { - cols := allOperatorWithArtifactNodeResultCols() - for i, col := range cols { - cols[i] = fmt.Sprintf("%s.%s", OperatorWithArtifactNodeResultTable, col) - } - - return strings.Join(cols, ",") -} - -func allOperatorWithArtifactNodeResultCols() []string { - return []string{ - OperatorWithArtifactNodeResultID, - OperatorWithArtifactNodeResultOperatorExecState, - OperatorWithArtifactNodeResultArtifactID, - OperatorWithArtifactNodeResultMetadata, - OperatorWithArtifactNodeResultContentPath, - OperatorWithArtifactNodeResultArtifactExecState, - } -} diff --git a/src/golang/lib/models/views/merged_node.go b/src/golang/lib/models/views/operator_with_artifact_node.go similarity index 96% rename from src/golang/lib/models/views/merged_node.go rename to src/golang/lib/models/views/operator_with_artifact_node.go index 5cf2687b67..c8d77c1f42 100644 --- a/src/golang/lib/models/views/merged_node.go +++ b/src/golang/lib/models/views/operator_with_artifact_node.go @@ -10,7 +10,7 @@ import ( ) const ( - OperatorWithArtifactNodeView = "merged_node" + OperatorWithArtifactNodeView = "operator_with_artifact_node" OperatorWithArtifactNodeID = "id" OperatorWithArtifactNodeDagID = "dag_id" OperatorWithArtifactNodeArtifactID = "artifact_id" diff --git a/src/golang/lib/models/views/operator_with_artifact_node_result.go b/src/golang/lib/models/views/operator_with_artifact_node_result.go new file mode 100644 index 0000000000..c7383d64d0 --- /dev/null +++ b/src/golang/lib/models/views/operator_with_artifact_node_result.go @@ -0,0 +1,64 @@ +package views + +import ( + "fmt" + "strings" + + "github.com/aqueducthq/aqueduct/lib/models/shared" + "github.com/google/uuid" +) + +const ( + OperatorWithArtifactResultNodeTable = "operator_with_artifact_node_result" + + // OperatorWithArtifactResultNode table column names + OperatorWithArtifactResultNodeID = "id" // operator result ID + OperatorWithArtifactResultNodeArtifactResultID = "artifact_result_id" + OperatorWithArtifactResultNodeOperatorID = "operator_id" + OperatorWithArtifactResultNodeArtifactID = "artifact_id" + OperatorWithArtifactResultNodeOperatorResultExecState = "operator_result_exec_state" + OperatorWithArtifactResultNodeMetadata = "metadata" + OperatorWithArtifactResultNodeContentPath = "content_path" + OperatorWithArtifactResultNodeArtifactResultExecState = "artifact_result_exec_state" +) + +// An OperatorWithArtifactResultNode maps to the merged_node_result table. +type OperatorWithArtifactResultNode struct { + ID uuid.UUID `db:"id" json:"id"` + OperatorID uuid.UUID `db:"operator_id" json:"operator_id"` + OperatorResultExecState shared.NullExecutionState `db:"operator_result_exec_state" json:"operator_result_exec_state"` + ArtifactID uuid.UUID `db:"artifact_id" json:"artifact_id"` + ArtifactResultID uuid.UUID `db:"artifact_result_id" json:"artifact_result_id"` + Metadata shared.NullArtifactResultMetadata `db:"metadata" json:"metadata"` + ContentPath string `db:"content_path" json:"content_path"` + ArtifactResultExecState shared.NullExecutionState `db:"artifact_result_exec_state" json:"artifact_result_exec_state"` +} + +// OperatorWithArtifactResultNodeCols returns a comma-separated string of all OperatorWithArtifactResultNode columns. +func OperatorWithArtifactResultNodeCols() string { + return strings.Join(allOperatorWithArtifactResultNodeCols(), ",") +} + +// OperatorWithArtifactResultNodeColsWithPrefix returns a comma-separated string of all +// OperatorWithArtifactResultNode columns prefixed by the table name. +func OperatorWithArtifactResultNodeColsWithPrefix() string { + cols := allOperatorWithArtifactResultNodeCols() + for i, col := range cols { + cols[i] = fmt.Sprintf("%s.%s", OperatorWithArtifactResultNodeTable, col) + } + + return strings.Join(cols, ",") +} + +func allOperatorWithArtifactResultNodeCols() []string { + return []string{ + OperatorWithArtifactResultNodeID, + OperatorWithArtifactResultNodeOperatorResultExecState, + OperatorWithArtifactResultNodeOperatorID, + OperatorWithArtifactResultNodeArtifactID, + OperatorWithArtifactResultNodeArtifactResultID, + OperatorWithArtifactResultNodeMetadata, + OperatorWithArtifactResultNodeContentPath, + OperatorWithArtifactResultNodeArtifactResultExecState, + } +} diff --git a/src/golang/lib/repos/operator_result.go b/src/golang/lib/repos/operator_result.go index 438f7bcd86..31d0a3b7ab 100644 --- a/src/golang/lib/repos/operator_result.go +++ b/src/golang/lib/repos/operator_result.go @@ -30,8 +30,8 @@ type operatorResultReader interface { // GetByDAGResultBatch returns all OperatorResults for the DAGResults specified. GetByDAGResultBatch(ctx context.Context, dagResultIDs []uuid.UUID, DB database.Database) ([]models.OperatorResult, error) - // GetOperatorWithArtifactNodeByOperatorNameAndWorkflow returns the OperatorWithArtifactNode for the Workflow and Operator specified. - GetOperatorWithArtifactNodeByOperatorNameAndWorkflow(ctx context.Context, dagResultID, operatorName string, workflowID uuid.UUID, DB database.Database) (*models.OperatorWithArtifactNodeResult, error) + // GetOperatorWithArtifactResultNodesByOperatorNameAndWorkflow returns the OperatorWithArtifactNode for the Workflow and Operator specified. + GetOperatorWithArtifactResultNodesByOperatorNameAndWorkflow(ctx context.Context, operatorName string, workflowID uuid.UUID, DB database.Database) ([]views.OperatorWithArtifactResultNode, error) // GetCheckStatusByArtifactBatch returns an OperatorResultStatus for all OperatorResults // associated with a Check Operator where the Operator has incoming DAGEdge diff --git a/src/golang/lib/repos/sqlite/operator.go b/src/golang/lib/repos/sqlite/operator.go index 7ad571dd24..20a646f1cf 100644 --- a/src/golang/lib/repos/sqlite/operator.go +++ b/src/golang/lib/repos/sqlite/operator.go @@ -200,7 +200,7 @@ func (*operatorReader) GetOperatorWithArtifactNodeBatch(ctx context.Context, IDs return getOperatorWithArtifactNodes(ctx, DB, query, args...) } -func (r *operatorReader) GetOperatorWithArtifactNodeByArtifactId(ctx context.Context, artifactID uuid.UUID, DB database.Database) (*views.OperatorWithArtifactNode, error) { +func (r *operatorReader) GetOperatorWithArtifactByArtifactIdNode(ctx context.Context, artifactID uuid.UUID, DB database.Database) (*views.OperatorWithArtifactNode, error) { nodes, err := r.GetOperatorWithArtifactNodeBatch(ctx, []uuid.UUID{artifactID}, DB) if err != nil { return nil, err @@ -208,9 +208,9 @@ func (r *operatorReader) GetOperatorWithArtifactNodeByArtifactId(ctx context.Con return &nodes[0], nil } -func (*operatorReader) GetOperatorWithArtifactNodeByArtifactIdBatch(ctx context.Context, artifactIDs []uuid.UUID, DB database.Database) ([]views.OperatorWithArtifactNode, error) { - if len(IDs) == 0 { - return nil, errors.New("Provided empty IDs list.") +func (*operatorReader) GetOperatorWithArtifactByArtifactIdNodeBatch(ctx context.Context, artifactIDs []uuid.UUID, DB database.Database) ([]views.OperatorWithArtifactNode, error) { + if len(artifactIDs) == 0 { + return nil, errors.New("Provided empty artifact IDs list.") } query := fmt.Sprintf( @@ -222,7 +222,7 @@ func (*operatorReader) GetOperatorWithArtifactNodeByArtifactIdBatch(ctx context. views.OperatorWithArtifactNodeArtifactID, stmt_preparers.GenerateArgsList(len(artifactIDs), 1), ) - args := stmt_preparers.CastIdsListToInterfaceList(IDs) + args := stmt_preparers.CastIdsListToInterfaceList(artifactIDs) return getOperatorWithArtifactNodes(ctx, DB, query, args...) } diff --git a/src/golang/lib/repos/sqlite/operator_result.go b/src/golang/lib/repos/sqlite/operator_result.go index 38a2544a6a..fe3bfdef44 100644 --- a/src/golang/lib/repos/sqlite/operator_result.go +++ b/src/golang/lib/repos/sqlite/operator_result.go @@ -184,6 +184,44 @@ func (*operatorResultReader) GetStatusByDAGResultAndArtifactBatch( return statuses, err } +func (*operatorResultReader) GetOperatorWithArtifactResultNodesByOperatorNameAndWorkflow( + ctx context.Context, + operatorName string, + workflowID uuid.UUID, + DB database.Database, +) ([]views.OperatorWithArtifactResultNode, error) { + // For all workflow dags that belong to the workflow (identified by ID), + // get the workflow dag edges of the workflow dag. + // Get all operators with the operator name, get the operator ids and + // find the operator results of each operator (by id). + // Get all the artifact results by finding all workflow_dag_edges + // from operator by id to artifact result by artifact id. + query := `SELECT + operator_result.id, + operator.id AS operator_id, + operator_result.execution_state AS operator_result_exec_state, + artifact_result.artifact_id, + artifact_result.id AS artifact_result_id, + artifact_result.metadata, + artifact_result.content_path, + artifact_result.execution_state AS artifact_result_exec_state, + FROM operator, operator_result, artifact_result, workflow_dag, workflow_dag_edge + WHERE + workflow_dag.workflow_id = $1 + AND workflow_dag_edge.workflow_dag_id = workflow_dag.id + AND operator.name = $2 + AND operator_result.id = operator.id + AND workflow_dag_edge.from_id = operator.id + AND workflow_dag_edge.to_id = artifact_result.artifact_id + AND artifact_result.workflow_dag_result_id = workflow_dag_result.id;` + + args := []interface{}{workflowID, operatorName} + + var operatorWithArtifactResultNodes []views.OperatorWithArtifactResultNode + err := DB.Query(ctx, &operatorWithArtifactResultNodes, query, args...) + return operatorWithArtifactResultNodes, err +} + func (*operatorResultWriter) Create( ctx context.Context, dagResultID uuid.UUID, diff --git a/src/golang/lib/response/node.go b/src/golang/lib/response/node.go index 26289c6d3e..8e8a585d9b 100644 --- a/src/golang/lib/response/node.go +++ b/src/golang/lib/response/node.go @@ -43,12 +43,14 @@ func NewOperatorWithArtifactNodeFromDBObject(dbOperatorWithArtifactNode *views.O } } -type OperatorWithArtifactNodeResult struct { - // Operator ID +type OperatorWithArtifactResultNode struct { + // Operator Result ID ID uuid.UUID `json:"id"` - OperatorExecState *shared.ExecutionState `json:"operator_exec_state"` - + ArtifactResultID uuid.UUID `json:"artifact_result_id"` + OperatorID uuid.UUID `json:"operator_id"` ArtifactID uuid.UUID `json:"artifact_id"` + OperatorResultExecState *shared.ExecutionState `json:"operator_result_exec_state"` + ArtifactResultExecState *shared.ExecutionState `json:"artifact_result_exec_state"` SerializationType shared.ArtifactSerializationType `json:"serialization_type"` // If `ContentSerialized` is set, the content is small and we directly send @@ -59,32 +61,32 @@ type OperatorWithArtifactNodeResult struct { // one should send an additional request to fetch the content. ContentPath string `json:"content_path"` ContentSerialized *string `json:"content_serialized"` - - ArtifactExecState *shared.ExecutionState `json:"artifact_exec_state"` } -func NewOperatorWithArtifactNodeResultFromDBObject( - dbOperatorWithArtifactNodeResult *views.OperatorWithArtifactNodeResult, +func NewOperatorWithArtifactResultNodeFromDBObject( + dbOperatorWithArtifactResultNode *views.OperatorWithArtifactResultNode, content *string, -) *OperatorWithArtifactNodeResult { - result := &OperatorWithArtifactNodeResult{ - ID: dbOperatorWithArtifactNodeResult.ID, - ArtifactID: dbOperatorWithArtifactNodeResult.ArtifactID, - SerializationType: dbOperatorWithArtifactNodeResult.Metadata.SerializationType, - ContentPath: dbOperatorWithArtifactNodeResult.ContentPath, +) *OperatorWithArtifactResultNode { + result := &OperatorWithArtifactResultNode{ + ID: dbOperatorWithArtifactResultNode.ID, + ArtifactResultID: dbOperatorWithArtifactResultNode.ArtifactResultID, + OperatorID: dbOperatorWithArtifactResultNode.OperatorID, + ArtifactID: dbOperatorWithArtifactResultNode.ArtifactID, + SerializationType: dbOperatorWithArtifactResultNode.Metadata.SerializationType, + ContentPath: dbOperatorWithArtifactResultNode.ContentPath, ContentSerialized: content, } - if !dbOperatorWithArtifactNodeResult.OperatorExecState.IsNull { + if !dbOperatorWithArtifactResultNode.OperatorResultExecState.IsNull { // make a copy of execState's value - execStateVal := dbOperatorWithArtifactNodeResult.OperatorExecState.ExecutionState - result.OperatorExecState = &execStateVal + execStateVal := dbOperatorWithArtifactResultNode.OperatorResultExecState.ExecutionState + result.OperatorResultExecState = &execStateVal } - if !dbOperatorWithArtifactNodeResult.ArtifactExecState.IsNull { + if !dbOperatorWithArtifactResultNode.ArtifactResultExecState.IsNull { // make a copy of execState's value - execStateVal := dbOperatorWithArtifactNodeResult.ArtifactExecState.ExecutionState - result.ArtifactExecState = &execStateVal + execStateVal := dbOperatorWithArtifactResultNode.ArtifactResultExecState.ExecutionState + result.ArtifactResultExecState = &execStateVal } return result @@ -251,8 +253,8 @@ type NodeResults struct { Operators []OperatorResult `json:"operators"` Artifacts []ArtifactResult `json:"artifacts"` // TODO: ENG-2987 Create separate sections for Metrics/Checks - // Metrics []OperatorWithArtifactNodeResult `json:"metrics"` - // Checks []OperatorWithArtifactNodeResult `json:"checks"` + // Metrics []OperatorWithArtifactResultNode `json:"metrics"` + // Checks []OperatorWithArtifactResultNode `json:"checks"` } func NewNodeResultsFromDBObjects( From 8c6dbf84cfabc0121066f4875b7097f3eec4e4b2 Mon Sep 17 00:00:00 2001 From: Eunice Chan Date: Fri, 2 Jun 2023 14:23:18 -0700 Subject: [PATCH 3/6] Routes work --- integration_tests/backend/test_reads.py | 1257 +++++++++-------- sdk/aqueduct/backend/api_client.py | 9 +- sdk/aqueduct/models/response_models.py | 51 + .../handler/v2/node_check_results_get.go | 18 +- .../handler/v2/node_metric_results_get.go | 18 +- src/golang/lib/repos/sqlite/operator.go | 6 +- .../lib/repos/sqlite/operator_result.go | 7 +- 7 files changed, 737 insertions(+), 629 deletions(-) diff --git a/integration_tests/backend/test_reads.py b/integration_tests/backend/test_reads.py index 18c3441ddc..b0e7084624 100644 --- a/integration_tests/backend/test_reads.py +++ b/integration_tests/backend/test_reads.py @@ -5,6 +5,7 @@ import pytest import requests +from aqueduct.constants.enums import ArtifactType import utils from aqueduct.constants.enums import RuntimeType from aqueduct.models.response_models import ( @@ -16,6 +17,7 @@ GetNodeResultContentResponse, GetOperatorResultResponse, GetOperatorWithArtifactNodeResponse, + GetOperatorWithArtifactNodeResultResponse, ) from aqueduct_executor.operators.utils.enums import JobType from exec_state import assert_exec_state @@ -52,11 +54,13 @@ class TestBackend: GET_NODE_METRIC_RESULT_CONTENT_TEMPLATE = ( "/api/v2/workflow/%s/dag/%s/node/metric/%s/result/%s/content" ) + GET_NODE_METRIC_RESULTS_TEMPLATE = "/api/v2/workflow/%s/dag/%s/node/metric/%s/results" GET_NODE_CHECK_TEMPLATE = "/api/v2/workflow/%s/dag/%s/node/check/%s" GET_NODE_CHECK_RESULT_CONTENT_TEMPLATE = ( "/api/v2/workflow/%s/dag/%s/node/check/%s/result/%s/content" ) + GET_NODE_CHECK_RESULTS_TEMPLATE = "/api/v2/workflow/%s/dag/%s/node/check/%s/results" # V1 LIST_WORKFLOW_SAVED_OBJECTS_TEMPLATE = "/api/workflow/%s/objects" @@ -100,13 +104,13 @@ def setup_class(cls): for flow_id, n_runs in cls.flows.values(): utils.wait_for_flow_runs(cls.client, flow_id, n_runs) - @classmethod - def teardown_class(cls): - for flow_id, _ in cls.flows.values(): - utils.delete_flow(cls.client, flow_id) + # @classmethod + # def teardown_class(cls): + # for flow_id, _ in cls.flows.values(): + # utils.delete_flow(cls.client, flow_id) - for flow_id, _ in cls.running_flows.values(): - utils.delete_flow(cls.client, flow_id) + # for flow_id, _ in cls.running_flows.values(): + # utils.delete_flow(cls.client, flow_id) @classmethod def response(cls, endpoint, additional_headers): @@ -127,474 +131,562 @@ def post_response(cls, endpoint, additional_headers={}): r = requests.post(url, headers=headers) return r - def test_endpoint_list_workflow_tables(self): - endpoint = self.LIST_WORKFLOW_SAVED_OBJECTS_TEMPLATE % self.flows["changing_saves"][0] - data = self.get_response(endpoint).json()["object_details"] - - assert len(data) == 3 - - # table_name, update_mode - data_set = set( - [ - ("table_1", "append"), - ("table_1", "replace"), - ("table_2", "replace"), - ] - ) - - print(data) - assert ( - set( - [ - (item["spec"]["parameters"]["table"], item["spec"]["parameters"]["update_mode"]) - for item in data - ] - ) - == data_set - ) - - # Check all in same resource - assert len(set([item["resource_name"] for item in data])) == 1 - assert len(set([item["spec"]["service"] for item in data])) == 1 - - def test_endpoint_delete_resource(self): - resource_name = f"test_delete_resource_{uuid.uuid4().hex[:8]}" - - # Check resource did not exist - data = self.get_response(self.LIST_INTEGRATIONS_TEMPLATE).json() - assert resource_name not in set([resource["name"] for resource in data]) - - # Create resource - status = self.post_response( - self.CONNECT_INTEGRATION_TEMPLATE, - additional_headers={ - "resource-name": resource_name, - "resource-service": "SQLite", - "resource-config": json.dumps({"database": self.DEMO_DB_PATH}), - }, - ).status_code - assert status == 200 - - # Check resource created - data = self.get_response(self.LIST_INTEGRATIONS_TEMPLATE).json() - resource_data = {resource["name"]: resource["id"] for resource in data} - assert resource_name in set(resource_data.keys()) - - # Delete resource - status = self.post_response( - self.DELETE_INTEGRATION_TEMPLATE % resource_data[resource_name] - ).status_code - assert status == 200 - - # Check resource does not exist - data = self.get_response(self.LIST_INTEGRATIONS_TEMPLATE).json() - assert resource_name not in set([resource["name"] for resource in data]) - - def test_endpoint_test_resource(self): - resp = self.get_response(self.GET_TEST_INTEGRATION_TEMPLATE % self.resource.id()) - assert resp.ok - - def test_endpoint_get_workflow_dag_result_with_failure(self): - flow_id = self.flows["flow_with_failure"][0] - flow = self.client.flow(flow_id) - runs = flow.list_runs() - resp = self.get_response( - self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) - ).json() - assert_exec_state(resp["result"]["exec_state"], "failed") - # operators - operators = resp["operators"] - assert len(operators) == 3 - for op in operators.values(): - name = op["name"] - exec_state = op["result"]["exec_state"] - - if "query" in name: # extract - assert_exec_state(exec_state, "succeeded") - elif name == "bad_op": - assert_exec_state(exec_state, "failed") - elif name == "bad_op_downstream": - assert_exec_state(exec_state, "canceled") - else: - raise Exception(f"unexpected operator name {name}") - - # artifacts - artifacts = resp["artifacts"] - assert len(artifacts) == 3 - for artf in artifacts.values(): - name = artf["name"] - exec_state = artf["result"]["exec_state"] - - if "query" in name: - assert_exec_state(exec_state, "succeeded") - elif name == "bad_op artifact": - assert_exec_state(exec_state, "canceled") - elif name == "bad_op_downstream artifact": - assert_exec_state(exec_state, "canceled") - else: - raise Exception(f"unexpected operator name {name}") - - def test_endpoint_get_workflow_dag_result_with_metrics_and_checks(self): + # def test_endpoint_list_workflow_tables(self): + # endpoint = self.LIST_WORKFLOW_SAVED_OBJECTS_TEMPLATE % self.flows["changing_saves"][0] + # data = self.get_response(endpoint).json()["object_details"] + + # assert len(data) == 3 + + # # table_name, update_mode + # data_set = set( + # [ + # ("table_1", "append"), + # ("table_1", "replace"), + # ("table_2", "replace"), + # ] + # ) + + # assert ( + # set( + # [ + # (item["spec"]["parameters"]["table"], item["spec"]["parameters"]["update_mode"]) + # for item in data + # ] + # ) + # == data_set + # ) + + # # Check all in same resource + # assert len(set([item["resource_name"] for item in data])) == 1 + # assert len(set([item["spec"]["service"] for item in data])) == 1 + + # def test_endpoint_delete_resource(self): + # resource_name = f"test_delete_resource_{uuid.uuid4().hex[:8]}" + + # # Check resource did not exist + # data = self.get_response(self.LIST_INTEGRATIONS_TEMPLATE).json() + # assert resource_name not in set([resource["name"] for resource in data]) + + # # Create resource + # status = self.post_response( + # self.CONNECT_INTEGRATION_TEMPLATE, + # additional_headers={ + # "resource-name": resource_name, + # "resource-service": "SQLite", + # "resource-config": json.dumps({"database": self.DEMO_DB_PATH}), + # }, + # ).status_code + # assert status == 200 + + # # Check resource created + # data = self.get_response(self.LIST_INTEGRATIONS_TEMPLATE).json() + # resource_data = {resource["name"]: resource["id"] for resource in data} + # assert resource_name in set(resource_data.keys()) + + # # Delete resource + # status = self.post_response( + # self.DELETE_INTEGRATION_TEMPLATE % resource_data[resource_name] + # ).status_code + # assert status == 200 + + # # Check resource does not exist + # data = self.get_response(self.LIST_INTEGRATIONS_TEMPLATE).json() + # assert resource_name not in set([resource["name"] for resource in data]) + + # def test_endpoint_test_resource(self): + # resp = self.get_response(self.GET_TEST_INTEGRATION_TEMPLATE % self.resource.id()) + # assert resp.ok + + # def test_endpoint_get_workflow_dag_result_with_failure(self): + # flow_id = self.flows["flow_with_failure"][0] + # flow = self.client.flow(flow_id) + # runs = flow.list_runs() + # resp = self.get_response( + # self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) + # ).json() + # assert_exec_state(resp["result"]["exec_state"], "failed") + # # operators + # operators = resp["operators"] + # assert len(operators) == 3 + # for op in operators.values(): + # name = op["name"] + # exec_state = op["result"]["exec_state"] + + # if "query" in name: # extract + # assert_exec_state(exec_state, "succeeded") + # elif name == "bad_op": + # assert_exec_state(exec_state, "failed") + # elif name == "bad_op_downstream": + # assert_exec_state(exec_state, "canceled") + # else: + # raise Exception(f"unexpected operator name {name}") + + # # artifacts + # artifacts = resp["artifacts"] + # assert len(artifacts) == 3 + # for artf in artifacts.values(): + # name = artf["name"] + # exec_state = artf["result"]["exec_state"] + + # if "query" in name: + # assert_exec_state(exec_state, "succeeded") + # elif name == "bad_op artifact": + # assert_exec_state(exec_state, "canceled") + # elif name == "bad_op_downstream artifact": + # assert_exec_state(exec_state, "canceled") + # else: + # raise Exception(f"unexpected operator name {name}") + + # def test_endpoint_get_workflow_dag_result_with_metrics_and_checks(self): + # flow_id = self.flows["flow_with_metrics_and_checks"][0] + # flow = self.client.flow(flow_id) + # runs = flow.list_runs() + # resp = self.get_response( + # self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) + # ).json() + # assert_exec_state(resp["result"]["exec_state"], "succeeded") + + # # operators + # operators = resp["operators"] + # assert len(operators) == 3 + # for op in operators.values(): + # name = op["name"] + # exec_state = op["result"]["exec_state"] + # if "query" in name or name == "size" or name == "check": # extract + # assert_exec_state(exec_state, "succeeded") + # else: + # raise Exception(f"unexpected operator name {name}") + + # # artifacts + # artifacts = resp["artifacts"] + # assert len(artifacts) == 3 + # for artf in artifacts.values(): + # name = artf["name"] + # exec_state = artf["result"]["exec_state"] + # value = artf["result"]["content_serialized"] + + # if "query" in name: + # assert_exec_state(exec_state, "succeeded") + # elif name == "size artifact": + # assert_exec_state(exec_state, "succeeded") + # assert int(value) > 0 + # elif name == "check artifact": + # assert_exec_state(exec_state, "succeeded") + # assert value == "true" + # else: + # raise Exception(f"unexpected operator name {name}") + + # def test_endpoint_get_workflow_dag_result_on_flow_with_sleep(self): + # flow_id = self.running_flows["flow_with_sleep"][0] + # flow = self.client.flow(flow_id) + # runs = flow.list_runs() + # resp = self.get_response( + # self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) + # ).json() + # assert_exec_state(resp["result"]["exec_state"], "pending") + + # # operators + # operators = resp["operators"] + # assert len(operators) == 2 + # for op in operators.values(): + # name = op["name"] + # exec_state = op["result"]["exec_state"] + # if "query" in name: # extract + # assert_exec_state(exec_state, "succeeded") + # elif name == "sleeping_op": + # assert_exec_state(exec_state, "pending") + # else: + # raise Exception(f"unexpected operator name {name}") + + # # artifacts + # artifacts = resp["artifacts"] + # assert len(artifacts) == 2 + # for artf in artifacts.values(): + # name = artf["name"] + # exec_state = artf["result"]["exec_state"] + + # if "query" in name: + # assert_exec_state(exec_state, "succeeded") + # elif name == "sleeping_op artifact": + # assert_exec_state(exec_state, "pending") + # else: + # raise Exception(f"unexpected operator name {name}") + + # def test_endpoint_list_artifact_results_with_metrics_and_checks(self): + # flow_id, num_runs = self.flows["flow_with_metrics_and_checks"] + # flow = self.client.flow(flow_id) + # runs = flow.list_runs() + # resp = self.get_response( + # self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) + # ).json() + + # # artifacts + # artifacts = resp["artifacts"] + # assert len(artifacts) == 3 + # for artf in artifacts.values(): + # name = artf["name"] + # id = artf["id"] + # resp = self.get_response(self.LIST_ARTIFACT_RESULTS_TEMPLATE % (flow_id, id)).json() + # results = resp["results"] + # assert len(results) == num_runs + + # for result in results: + # exec_state = result["exec_state"] + # value = result["content_serialized"] + # assert_exec_state(exec_state, "succeeded") + + # if "query" in name: + # assert value is None + # elif name == "size artifact": + # assert int(value) > 0 + # elif name == "check artifact": + # assert value == "true" + + # def test_endpoint_workflows_get(self): + # resp = self.get_response(self.GET_WORKFLOWS_TEMPLATE) + # resp = resp.json() + + # if len(resp) > 0: + # keys = [ + # "id", + # "user_id", + # "name", + # "description", + # "schedule", + # "created_at", + # "retention_policy", + # "notification_settings", + # ] + + # user_id = resp[0]["user_id"] + + # for v2_workflow in resp: + # for key in keys: + # assert key in v2_workflow + # assert v2_workflow["user_id"] == user_id + + # def test_endpoint_workflow_dags_get(self): + # flow_id, _ = self.flows["flow_with_metrics_and_checks"] + # resp = self.get_response(self.GET_DAGS_TEMPLATE % flow_id) + # resp = resp.json() + + # assert len(resp) == 2 + # for dag_dict in resp: + # dag = GetDagResponse(**dag_dict) + # assert dag.workflow_id == flow_id + # assert dag.created_at != "" + # assert dag.engine_config.type == RuntimeType.AQUEDUCT + + # def test_endpoint_dag_results_get(self): + # flow_id, n_runs = self.flows["flow_with_metrics_and_checks"] + # resp = self.get_response(self.GET_DAG_RESULTS_TEMPLATE % flow_id).json() + + # assert len(resp) == n_runs + + # def check_structure(resp, all_succeeded=False): + # for result in resp: + # result = GetDagResultResponse(**result) + # if all_succeeded: + # assert result.exec_state.status == "succeeded" + # assert result.exec_state.failure_type == None + # assert result.exec_state.error == None + + # check_structure(resp, all_succeeded=True) + + # # Using the order parameter + # flow_id, n_runs = self.flows["flow_with_failure"] + # resp = self.get_response( + # self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status", + # ).json() + + # check_structure(resp) + # statuses = [result["exec_state"]["status"] for result in resp] + # sorted_statuses = sorted(statuses, reverse=True) # Descending order + # assert statuses == sorted_statuses + + # # Default is descending + # flow_id, n_runs = self.flows["flow_with_failure"] + # resp = self.get_response( + # self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status&order_descending=true", + # ).json() + + # check_structure(resp) + # descending_statuses = [result["exec_state"]["status"] for result in resp] + # assert statuses == descending_statuses + + # # Ascending works + # flow_id, n_runs = self.flows["flow_with_failure"] + # resp = self.get_response( + # self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status&order_descending=false", + # ).json() + + # check_structure(resp) + # ascending_statuses = [result["exec_state"]["status"] for result in resp] + # assert descending_statuses[::-1] == ascending_statuses + + # # Using the limit parameter + # resp = self.get_response( + # self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?limit=1", + # ).json() + + # check_structure(resp) + # assert len(resp) == 1 + + # # Using both the order and limit parameters + # resp = self.get_response( + # self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status&limit=1", + # ).json() + + # check_structure(resp) + # workflow_status = [result["exec_state"]["status"] for result in resp] + # assert len(workflow_status) == 1 + # workflow_status = workflow_status[0] + # assert workflow_status == sorted_statuses[0] + + # def test_endpoint_nodes_get(self): + # for flow_id, _ in [ + # self.flows["flow_with_metrics_and_checks"], + # self.flows["flow_with_multiple_operators"], + # ]: + # flow = self.client.flow(flow_id) + # workflow_resp = flow._get_workflow_resp() + # dag_id = list(workflow_resp.workflow_dags.keys())[0] + # resp = self.get_response(self.GET_NODES_TEMPLATE % (flow_id, dag_id)).json() + + # all_output_counts = [] + # for operator in resp["operators"]: + # result = GetNodeOperatorResponse(**operator) + # all_output_counts.append(len(result.outputs)) + # assert sum(all_output_counts) == len(all_output_counts) + # assert set(all_output_counts) == set([1]) + + # all_output_counts = [] + # for artifact in resp["artifacts"]: + # result = GetNodeArtifactResponse(**artifact) + # all_output_counts.append(len(result.outputs)) + # assert sum(all_output_counts) == len(all_output_counts) - 1 + # assert set(all_output_counts) == set([0, 1]) + + # def test_endpoint_nodes_results_get(self): + # for flow_id, _ in [ + # self.flows["flow_with_metrics_and_checks"], + # self.flows["flow_with_multiple_operators"], + # ]: + # flow = self.client.flow(flow_id) + # workflow_resp = flow._get_workflow_resp() + # dag_result_id = workflow_resp.workflow_dag_results[0].id + # resp = self.get_response( + # self.GET_NODES_RESULTS_TEMPLATE % (flow_id, dag_result_id) + # ).json() + # assert "operators" in resp.keys() + # assert "artifacts" in resp.keys() + # assert len(resp["operators"]) == len(resp["artifacts"]) + # for op in resp["operators"]: + # result = GetOperatorResultResponse(**op) + # result.exec_state.status == "succeeded" + # for artf in resp["artifacts"]: + # result = GetArtifactResultResponse(**artf) + # result.exec_state.status == "succeeded" + + # def test_endpoint_node_artifact_get(self): + # for flow_id, _ in [ + # self.flows["flow_with_metrics_and_checks"], + # self.flows["flow_with_multiple_operators"], + # ]: + # flow = self.client.flow(flow_id) + # workflow_resp = flow._get_workflow_resp() + # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + # dag_result_id = workflow_resp.workflow_dag_results[0].id + + # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + # flow_id, + # dag_result_id, + # ) + # artifact_ids = list(dag_result_resp.artifacts.keys()) + # artifact_id = str(artifact_ids[0]) + # all_output_counts = [] + # for artifact_id in artifact_ids: + # artifact_id = str(artifact_id) + # resp = self.get_response( + # self.GET_NODE_ARTIFACT_TEMPLATE % (flow_id, dag_id, artifact_id) + # ).json() + # result = GetNodeArtifactResponse(**resp) + # all_output_counts.append(len(result.outputs)) + # assert sum(all_output_counts) == len(all_output_counts) - 1 + # assert set(all_output_counts) == set([0, 1]) + + # def test_endpoint_node_artifact_result_content_get(self): + # for flow_id, _ in [ + # self.flows["flow_with_metrics_and_checks"], + # self.flows["flow_with_multiple_operators"], + # ]: + # flow = self.client.flow(flow_id) + # workflow_resp = flow._get_workflow_resp() + # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + # dag_result_id = workflow_resp.workflow_dag_results[0].id + + # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + # flow_id, + # dag_result_id, + # ) + # artifact_ids = list(dag_result_resp.artifacts.keys()) + # artifact_id = str(artifact_ids[0]) + + # resp = self.get_response( + # self.GET_NODE_ARTIFACT_RESULTS_TEMPLATE % (flow_id, dag_id, artifact_id) + # ).json() + # downstream_ids = [GetArtifactResultResponse(**result).id for result in resp] + # for downstream_id in downstream_ids: + # artifact_result_id = str(downstream_id) + # resp = self.get_response( + # self.GET_NODE_ARTIFACT_RESULT_CONTENT_TEMPLATE + # % (flow_id, dag_id, artifact_id, artifact_result_id) + # ) + # assert resp.ok + # resp_obj = GetNodeResultContentResponse(**resp.json()) + # # One of these should be successful (direct descendent of operator) + # assert not resp_obj.is_downsampled + # assert len(resp_obj.content) > 0 + + # def test_endpoint_node_artifact_results_get(self): + # for flow_id, _ in [ + # self.flows["flow_with_metrics_and_checks"], + # self.flows["flow_with_multiple_operators"], + # ]: + # flow = self.client.flow(flow_id) + # workflow_resp = flow._get_workflow_resp() + # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + # dag_result_id = workflow_resp.workflow_dag_results[0].id + + # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + # flow_id, + # dag_result_id, + # ) + # artifact_ids = list(dag_result_resp.artifacts.keys()) + # artifact_id = str(artifact_ids[0]) + + # resp = self.get_response( + # self.GET_NODE_ARTIFACT_RESULTS_TEMPLATE % (flow_id, dag_id, artifact_id) + # ).json() + # for result in resp: + # result = GetArtifactResultResponse(**result) + + # def test_endpoint_node_operator_get(self): + # for flow_id, _ in [ + # self.flows["flow_with_metrics_and_checks"], + # self.flows["flow_with_multiple_operators"], + # ]: + # flow = self.client.flow(flow_id) + # workflow_resp = flow._get_workflow_resp() + # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + # dag_result_id = workflow_resp.workflow_dag_results[0].id + + # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + # flow_id, + # dag_result_id, + # ) + # operator_ids = list(dag_result_resp.operators.keys()) + # operator_id = str(operator_ids[0]) + + # resp = self.get_response( + # self.GET_NODE_OPERATOR_TEMPLATE % (flow_id, dag_id, operator_id) + # ).json() + # result = GetNodeOperatorResponse(**resp) + # assert str(result.id) == operator_id + # assert result.dag_id == dag_id + + # def test_endpoint_node_operator_content_get(self): + # flow_id, _ = self.flows["flow_with_multiple_operators"] + # flow = self.client.flow(flow_id) + # workflow_resp = flow._get_workflow_resp() + # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + # dag_result_id = workflow_resp.workflow_dag_results[0].id + + # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + # flow_id, + # dag_result_id, + # ) + # operator_ids = list(dag_result_resp.operators.keys()) + # operator_id = str(operator_ids[0]) + + # resp = self.get_response( + # self.GET_NODE_OPERATOR_CONTENT_TEMPLATE % (flow_id, dag_id, operator_id) + # ) + # # The response is a form data. For now, we simply check the response's code. + # assert resp.ok + + # def test_endpoint_node_metric_get(self): + # flow_id, _ = self.flows["flow_with_metrics_and_checks"] + # flow = self.client.flow(flow_id) + # workflow_resp = flow._get_workflow_resp() + # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + # dag_result_id = workflow_resp.workflow_dag_results[0].id + + # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + # flow_id, + # dag_result_id, + # ) + # operator_ids = [ + # id + # for id in dag_result_resp.operators.keys() + # if dag_result_resp.operators[id].spec.metric + # ] + # operator_id = str(operator_ids[0]) + + # resp = self.get_response( + # self.GET_NODE_METRIC_TEMPLATE % (flow_id, dag_id, operator_id) + # ).json() + # result = GetOperatorWithArtifactNodeResponse(**resp) + # assert str(result.id) == operator_id + # assert result.dag_id == dag_id + # assert len(result.inputs) == 1 + # assert len(result.outputs) == 1 + + # def test_endpoint_node_metric_result_content_get(self): + # flow_id, _ = self.flows["flow_with_metrics_and_checks"] + # flow = self.client.flow(flow_id) + # workflow_resp = flow._get_workflow_resp() + # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + # dag_result_id = workflow_resp.workflow_dag_results[0].id + + # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + # flow_id, + # dag_result_id, + # ) + # operator_ids = [ + # id + # for id in dag_result_resp.operators.keys() + # if dag_result_resp.operators[id].spec.metric + # ] + # operator_id = str(operator_ids[0]) + + # resp = self.get_response( + # self.GET_NODE_METRIC_TEMPLATE % (flow_id, dag_id, operator_id) + # ).json() + + # result = GetOperatorWithArtifactNodeResponse(**resp) + + # artifact_id = result.artifact_id + + # resp = self.get_response( + # self.LIST_ARTIFACT_RESULTS_TEMPLATE % (flow_id, artifact_id) + # ).json() + # results = resp["results"] + # # One of these should be correct for the DAG run and can get result content. + # for artifact_result in results: + # resp = self.get_response( + # self.GET_NODE_METRIC_RESULT_CONTENT_TEMPLATE + # % (flow_id, dag_id, operator_id, artifact_result["id"]) + # ) + # assert resp.ok + # resp_obj = GetNodeResultContentResponse(**resp.json()) + # # One of these should be successful (direct descendent of operator) + # assert not resp_obj.is_downsampled + # assert len(resp_obj.content) > 0 + + def test_endpoint_node_metric_results_get(self): flow_id = self.flows["flow_with_metrics_and_checks"][0] flow = self.client.flow(flow_id) - runs = flow.list_runs() - resp = self.get_response( - self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) - ).json() - assert_exec_state(resp["result"]["exec_state"], "succeeded") - - # operators - operators = resp["operators"] - assert len(operators) == 3 - for op in operators.values(): - name = op["name"] - exec_state = op["result"]["exec_state"] - if "query" in name or name == "size" or name == "check": # extract - assert_exec_state(exec_state, "succeeded") - else: - raise Exception(f"unexpected operator name {name}") - - # artifacts - artifacts = resp["artifacts"] - assert len(artifacts) == 3 - for artf in artifacts.values(): - name = artf["name"] - exec_state = artf["result"]["exec_state"] - value = artf["result"]["content_serialized"] - - if "query" in name: - assert_exec_state(exec_state, "succeeded") - elif name == "size artifact": - assert_exec_state(exec_state, "succeeded") - assert int(value) > 0 - elif name == "check artifact": - assert_exec_state(exec_state, "succeeded") - assert value == "true" - else: - raise Exception(f"unexpected operator name {name}") - - def test_endpoint_get_workflow_dag_result_on_flow_with_sleep(self): - flow_id = self.running_flows["flow_with_sleep"][0] - flow = self.client.flow(flow_id) - runs = flow.list_runs() - resp = self.get_response( - self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) - ).json() - assert_exec_state(resp["result"]["exec_state"], "pending") - - # operators - operators = resp["operators"] - assert len(operators) == 2 - for op in operators.values(): - name = op["name"] - exec_state = op["result"]["exec_state"] - if "query" in name: # extract - assert_exec_state(exec_state, "succeeded") - elif name == "sleeping_op": - assert_exec_state(exec_state, "pending") - else: - raise Exception(f"unexpected operator name {name}") - - # artifacts - artifacts = resp["artifacts"] - assert len(artifacts) == 2 - for artf in artifacts.values(): - name = artf["name"] - exec_state = artf["result"]["exec_state"] - - if "query" in name: - assert_exec_state(exec_state, "succeeded") - elif name == "sleeping_op artifact": - assert_exec_state(exec_state, "pending") - else: - raise Exception(f"unexpected operator name {name}") - - def test_endpoint_list_artifact_results_with_metrics_and_checks(self): - flow_id, num_runs = self.flows["flow_with_metrics_and_checks"] - flow = self.client.flow(flow_id) - runs = flow.list_runs() - resp = self.get_response( - self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) - ).json() - - # artifacts - artifacts = resp["artifacts"] - assert len(artifacts) == 3 - for artf in artifacts.values(): - name = artf["name"] - id = artf["id"] - resp = self.get_response(self.LIST_ARTIFACT_RESULTS_TEMPLATE % (flow_id, id)).json() - results = resp["results"] - assert len(results) == num_runs - - for result in results: - exec_state = result["exec_state"] - value = result["content_serialized"] - assert_exec_state(exec_state, "succeeded") - - if "query" in name: - assert value is None - elif name == "size artifact": - assert int(value) > 0 - elif name == "check artifact": - assert value == "true" - - def test_endpoint_workflows_get(self): - resp = self.get_response(self.GET_WORKFLOWS_TEMPLATE) - resp = resp.json() - - if len(resp) > 0: - keys = [ - "id", - "user_id", - "name", - "description", - "schedule", - "created_at", - "retention_policy", - "notification_settings", - ] - - user_id = resp[0]["user_id"] - - for v2_workflow in resp: - for key in keys: - assert key in v2_workflow - assert v2_workflow["user_id"] == user_id - - def test_endpoint_workflow_dags_get(self): - flow_id, _ = self.flows["flow_with_metrics_and_checks"] - resp = self.get_response(self.GET_DAGS_TEMPLATE % flow_id) - resp = resp.json() - - assert len(resp) == 2 - for dag_dict in resp: - dag = GetDagResponse(**dag_dict) - assert dag.workflow_id == flow_id - assert dag.created_at != "" - assert dag.engine_config.type == RuntimeType.AQUEDUCT - - def test_endpoint_dag_results_get(self): - flow_id, n_runs = self.flows["flow_with_metrics_and_checks"] - resp = self.get_response(self.GET_DAG_RESULTS_TEMPLATE % flow_id).json() - - assert len(resp) == n_runs - - def check_structure(resp, all_succeeded=False): - for result in resp: - result = GetDagResultResponse(**result) - if all_succeeded: - assert result.exec_state.status == "succeeded" - assert result.exec_state.failure_type == None - assert result.exec_state.error == None - - check_structure(resp, all_succeeded=True) - - # Using the order parameter - flow_id, n_runs = self.flows["flow_with_failure"] - resp = self.get_response( - self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status", - ).json() - - check_structure(resp) - statuses = [result["exec_state"]["status"] for result in resp] - sorted_statuses = sorted(statuses, reverse=True) # Descending order - assert statuses == sorted_statuses - - # Default is descending - flow_id, n_runs = self.flows["flow_with_failure"] - resp = self.get_response( - self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status&order_descending=true", - ).json() - - check_structure(resp) - descending_statuses = [result["exec_state"]["status"] for result in resp] - assert statuses == descending_statuses - - # Ascending works - flow_id, n_runs = self.flows["flow_with_failure"] - resp = self.get_response( - self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status&order_descending=false", - ).json() - - check_structure(resp) - ascending_statuses = [result["exec_state"]["status"] for result in resp] - assert descending_statuses[::-1] == ascending_statuses - - # Using the limit parameter - resp = self.get_response( - self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?limit=1", - ).json() - - check_structure(resp) - assert len(resp) == 1 - - # Using both the order and limit parameters - resp = self.get_response( - self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status&limit=1", - ).json() - - check_structure(resp) - workflow_status = [result["exec_state"]["status"] for result in resp] - assert len(workflow_status) == 1 - workflow_status = workflow_status[0] - assert workflow_status == sorted_statuses[0] - - def test_endpoint_nodes_get(self): - for flow_id, _ in [ - self.flows["flow_with_metrics_and_checks"], - self.flows["flow_with_multiple_operators"], - ]: - flow = self.client.flow(flow_id) - workflow_resp = flow._get_workflow_resp() - dag_id = list(workflow_resp.workflow_dags.keys())[0] - resp = self.get_response(self.GET_NODES_TEMPLATE % (flow_id, dag_id)).json() - - all_output_counts = [] - for operator in resp["operators"]: - result = GetNodeOperatorResponse(**operator) - all_output_counts.append(len(result.outputs)) - assert sum(all_output_counts) == len(all_output_counts) - assert set(all_output_counts) == set([1]) - - all_output_counts = [] - for artifact in resp["artifacts"]: - result = GetNodeArtifactResponse(**artifact) - all_output_counts.append(len(result.outputs)) - assert sum(all_output_counts) == len(all_output_counts) - 1 - assert set(all_output_counts) == set([0, 1]) - - def test_endpoint_nodes_results_get(self): - for flow_id, _ in [ - self.flows["flow_with_metrics_and_checks"], - self.flows["flow_with_multiple_operators"], - ]: - flow = self.client.flow(flow_id) - workflow_resp = flow._get_workflow_resp() - dag_result_id = workflow_resp.workflow_dag_results[0].id - resp = self.get_response( - self.GET_NODES_RESULTS_TEMPLATE % (flow_id, dag_result_id) - ).json() - assert "operators" in resp.keys() - assert "artifacts" in resp.keys() - assert len(resp["operators"]) == len(resp["artifacts"]) - for op in resp["operators"]: - result = GetOperatorResultResponse(**op) - result.exec_state.status == "succeeded" - for artf in resp["artifacts"]: - result = GetArtifactResultResponse(**artf) - result.exec_state.status == "succeeded" - - def test_endpoint_node_artifact_get(self): - for flow_id, _ in [ - self.flows["flow_with_metrics_and_checks"], - self.flows["flow_with_multiple_operators"], - ]: - flow = self.client.flow(flow_id) - workflow_resp = flow._get_workflow_resp() - dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - dag_result_id = workflow_resp.workflow_dag_results[0].id - - dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - flow_id, - dag_result_id, - ) - artifact_ids = list(dag_result_resp.artifacts.keys()) - artifact_id = str(artifact_ids[0]) - all_output_counts = [] - for artifact_id in artifact_ids: - artifact_id = str(artifact_id) - resp = self.get_response( - self.GET_NODE_ARTIFACT_TEMPLATE % (flow_id, dag_id, artifact_id) - ).json() - result = GetNodeArtifactResponse(**resp) - all_output_counts.append(len(result.outputs)) - assert sum(all_output_counts) == len(all_output_counts) - 1 - assert set(all_output_counts) == set([0, 1]) - - def test_endpoint_node_artifact_result_content_get(self): - for flow_id, _ in [ - self.flows["flow_with_metrics_and_checks"], - self.flows["flow_with_multiple_operators"], - ]: - flow = self.client.flow(flow_id) - workflow_resp = flow._get_workflow_resp() - dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - dag_result_id = workflow_resp.workflow_dag_results[0].id - - dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - flow_id, - dag_result_id, - ) - artifact_ids = list(dag_result_resp.artifacts.keys()) - artifact_id = str(artifact_ids[0]) - - resp = self.get_response( - self.GET_NODE_ARTIFACT_RESULTS_TEMPLATE % (flow_id, dag_id, artifact_id) - ).json() - downstream_ids = [GetArtifactResultResponse(**result).id for result in resp] - for downstream_id in downstream_ids: - artifact_result_id = str(downstream_id) - resp = self.get_response( - self.GET_NODE_ARTIFACT_RESULT_CONTENT_TEMPLATE - % (flow_id, dag_id, artifact_id, artifact_result_id) - ) - assert resp.ok - resp_obj = GetNodeResultContentResponse(**resp.json()) - # One of these should be successful (direct descendent of operator) - assert not resp_obj.is_downsampled - assert len(resp_obj.content) > 0 - - def test_endpoint_node_artifact_results_get(self): - for flow_id, _ in [ - self.flows["flow_with_metrics_and_checks"], - self.flows["flow_with_multiple_operators"], - ]: - flow = self.client.flow(flow_id) - workflow_resp = flow._get_workflow_resp() - dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - dag_result_id = workflow_resp.workflow_dag_results[0].id - - dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - flow_id, - dag_result_id, - ) - artifact_ids = list(dag_result_resp.artifacts.keys()) - artifact_id = str(artifact_ids[0]) - - resp = self.get_response( - self.GET_NODE_ARTIFACT_RESULTS_TEMPLATE % (flow_id, dag_id, artifact_id) - ).json() - for result in resp: - result = GetArtifactResultResponse(**result) - - def test_endpoint_node_operator_get(self): - for flow_id, _ in [ - self.flows["flow_with_metrics_and_checks"], - self.flows["flow_with_multiple_operators"], - ]: - flow = self.client.flow(flow_id) - workflow_resp = flow._get_workflow_resp() - dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - dag_result_id = workflow_resp.workflow_dag_results[0].id - - dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - flow_id, - dag_result_id, - ) - operator_ids = list(dag_result_resp.operators.keys()) - operator_id = str(operator_ids[0]) - - resp = self.get_response( - self.GET_NODE_OPERATOR_TEMPLATE % (flow_id, dag_id, operator_id) - ).json() - result = GetNodeOperatorResponse(**resp) - assert str(result.id) == operator_id - assert result.dag_id == dag_id - - def test_endpoint_node_operator_content_get(self): - flow_id, _ = self.flows["flow_with_multiple_operators"] - flow = self.client.flow(flow_id) workflow_resp = flow._get_workflow_resp() dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id dag_result_id = workflow_resp.workflow_dag_results[0].id @@ -603,113 +695,88 @@ def test_endpoint_node_operator_content_get(self): flow_id, dag_result_id, ) - operator_ids = list(dag_result_resp.operators.keys()) - operator_id = str(operator_ids[0]) - + metric_artifact_id = None + for artifact_id, artifact in dag_result_resp.artifacts.items(): + if artifact.type == ArtifactType.NUMERIC: + metric_artifact_id = artifact_id + break resp = self.get_response( - self.GET_NODE_OPERATOR_CONTENT_TEMPLATE % (flow_id, dag_id, operator_id) - ) - # The response is a form data. For now, we simply check the response's code. - assert resp.ok - - def test_endpoint_node_metric_get(self): - flow_id, _ = self.flows["flow_with_metrics_and_checks"] - flow = self.client.flow(flow_id) - workflow_resp = flow._get_workflow_resp() - dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - dag_result_id = workflow_resp.workflow_dag_results[0].id - - dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - flow_id, - dag_result_id, - ) - operator_ids = [ - id - for id in dag_result_resp.operators.keys() - if dag_result_resp.operators[id].spec.metric - ] - operator_id = str(operator_ids[0]) - - resp = self.get_response( - self.GET_NODE_METRIC_TEMPLATE % (flow_id, dag_id, operator_id) - ).json() - result = GetOperatorWithArtifactNodeResponse(**resp) - assert str(result.id) == operator_id - assert result.dag_id == dag_id - assert len(result.inputs) == 1 - assert len(result.outputs) == 1 - - def test_endpoint_node_metric_result_content_get(self): - flow_id, _ = self.flows["flow_with_metrics_and_checks"] - flow = self.client.flow(flow_id) - workflow_resp = flow._get_workflow_resp() - dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - dag_result_id = workflow_resp.workflow_dag_results[0].id - - dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - flow_id, - dag_result_id, - ) - operator_ids = [ - id - for id in dag_result_resp.operators.keys() - if dag_result_resp.operators[id].spec.metric - ] - operator_id = str(operator_ids[0]) - - resp = self.get_response( - self.GET_NODE_METRIC_TEMPLATE % (flow_id, dag_id, operator_id) - ).json() - - result = GetOperatorWithArtifactNodeResponse(**resp) - - artifact_id = result.artifact_id - - resp = self.get_response( - self.LIST_ARTIFACT_RESULTS_TEMPLATE % (flow_id, artifact_id) - ).json() - results = resp["results"] - # One of these should be correct for the DAG run and can get result content. - for artifact_result in results: - resp = self.get_response( - self.GET_NODE_METRIC_RESULT_CONTENT_TEMPLATE - % (flow_id, dag_id, operator_id, artifact_result["id"]) - ) - assert resp.ok - resp_obj = GetNodeResultContentResponse(**resp.json()) - # One of these should be successful (direct descendent of operator) - assert not resp_obj.is_downsampled - assert len(resp_obj.content) > 0 - - def test_endpoint_node_check_get(self): - flow_id, _ = self.flows["flow_with_metrics_and_checks"] - flow = self.client.flow(flow_id) - workflow_resp = flow._get_workflow_resp() - dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - dag_result_id = workflow_resp.workflow_dag_results[0].id - - dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - flow_id, - dag_result_id, - ) - operator_ids = [ - id - for id in dag_result_resp.operators.keys() - if dag_result_resp.operators[id].spec.check - ] - operator_id = str(operator_ids[0]) - - resp = self.get_response( - self.GET_NODE_CHECK_TEMPLATE % (flow_id, dag_id, operator_id) + self.GET_NODE_METRIC_RESULTS_TEMPLATE % (flow_id, dag_id, metric_artifact_id) ).json() - result = GetOperatorWithArtifactNodeResponse(**resp) - assert str(result.id) == operator_id - assert result.dag_id == dag_id - assert len(result.inputs) == 1 - assert len(result.outputs) == 0 - - def test_endpoint_node_check_result_content_get(self): - flow_id, _ = self.flows["flow_with_metrics_and_checks"] + for result in resp: + result = GetOperatorWithArtifactNodeResultResponse(**result) + + # def test_endpoint_node_check_get(self): + # flow_id, _ = self.flows["flow_with_metrics_and_checks"] + # flow = self.client.flow(flow_id) + # workflow_resp = flow._get_workflow_resp() + # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + # dag_result_id = workflow_resp.workflow_dag_results[0].id + + # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + # flow_id, + # dag_result_id, + # ) + # operator_ids = [ + # id + # for id in dag_result_resp.operators.keys() + # if dag_result_resp.operators[id].spec.check + # ] + # operator_id = str(operator_ids[0]) + + # resp = self.get_response( + # self.GET_NODE_CHECK_TEMPLATE % (flow_id, dag_id, operator_id) + # ).json() + # result = GetOperatorWithArtifactNodeResponse(**resp) + # assert str(result.id) == operator_id + # assert result.dag_id == dag_id + # assert len(result.inputs) == 1 + # assert len(result.outputs) == 0 + + # def test_endpoint_node_check_result_content_get(self): + # flow_id, _ = self.flows["flow_with_metrics_and_checks"] + # flow = self.client.flow(flow_id) + # workflow_resp = flow._get_workflow_resp() + # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + # dag_result_id = workflow_resp.workflow_dag_results[0].id + + # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + # flow_id, + # dag_result_id, + # ) + # operator_ids = [ + # id + # for id in dag_result_resp.operators.keys() + # if dag_result_resp.operators[id].spec.check + # ] + # operator_id = str(operator_ids[0]) + + # resp = self.get_response( + # self.GET_NODE_CHECK_TEMPLATE % (flow_id, dag_id, operator_id) + # ).json() + + # result = GetOperatorWithArtifactNodeResponse(**resp) + + # artifact_id = result.artifact_id + + # resp = self.get_response( + # self.LIST_ARTIFACT_RESULTS_TEMPLATE % (flow_id, artifact_id) + # ).json() + # results = resp["results"] + # # One of these should be correct for the DAG run and can get result content. + # for artifact_result in results: + # resp = self.get_response( + # self.GET_NODE_CHECK_RESULT_CONTENT_TEMPLATE + # % (flow_id, dag_id, operator_id, artifact_result["id"]) + # ) + # assert resp.ok + # resp_obj = GetNodeResultContentResponse(**resp.json()) + # # One of these should be successful (direct descendent of operator) + # assert not resp_obj.is_downsampled + # assert len(resp_obj.content) > 0 + + def test_endpoint_node_check_results_get(self): + flow_id = self.flows["flow_with_metrics_and_checks"][0] flow = self.client.flow(flow_id) workflow_resp = flow._get_workflow_resp() dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id @@ -719,33 +786,13 @@ def test_endpoint_node_check_result_content_get(self): flow_id, dag_result_id, ) - operator_ids = [ - id - for id in dag_result_resp.operators.keys() - if dag_result_resp.operators[id].spec.check - ] - operator_id = str(operator_ids[0]) - - resp = self.get_response( - self.GET_NODE_CHECK_TEMPLATE % (flow_id, dag_id, operator_id) - ).json() - - result = GetOperatorWithArtifactNodeResponse(**resp) - - artifact_id = result.artifact_id - + check_artifact_id = None + for artifact_id, artifact in dag_result_resp.artifacts.items(): + if artifact.type == ArtifactType.BOOL: + check_artifact_id = artifact_id + break resp = self.get_response( - self.LIST_ARTIFACT_RESULTS_TEMPLATE % (flow_id, artifact_id) + self.GET_NODE_CHECK_RESULTS_TEMPLATE % (flow_id, dag_id, check_artifact_id) ).json() - results = resp["results"] - # One of these should be correct for the DAG run and can get result content. - for artifact_result in results: - resp = self.get_response( - self.GET_NODE_CHECK_RESULT_CONTENT_TEMPLATE - % (flow_id, dag_id, operator_id, artifact_result["id"]) - ) - assert resp.ok - resp_obj = GetNodeResultContentResponse(**resp.json()) - # One of these should be successful (direct descendent of operator) - assert not resp_obj.is_downsampled - assert len(resp_obj.content) > 0 + for result in resp: + result = GetOperatorWithArtifactNodeResultResponse(**result) \ No newline at end of file diff --git a/sdk/aqueduct/backend/api_client.py b/sdk/aqueduct/backend/api_client.py index e2b8ad4425..641bfb51ba 100644 --- a/sdk/aqueduct/backend/api_client.py +++ b/sdk/aqueduct/backend/api_client.py @@ -1,4 +1,4 @@ -import datetime +from dateutil import parser import io import json import uuid @@ -610,11 +610,8 @@ def get_workflow(self, flow_id: str) -> GetWorkflowV1Response: WorkflowDagResultResponse( id=dag_result.id, created_at=int( - datetime.datetime.strptime( - resp_dags[str(dag_result.dag_id)].created_at[:-4], - "%Y-%m-%dT%H:%M:%S.%f" - if resp_dags[str(dag_result.dag_id)].created_at[-1] == "Z" - else "%Y-%m-%dT%H:%M:%S.%f%z", + parser.parse( + resp_dags[str(dag_result.dag_id)].created_at ).timestamp() ), status=dag_result.exec_state.status, diff --git a/sdk/aqueduct/models/response_models.py b/sdk/aqueduct/models/response_models.py index 7aafebf1e5..6e28c8698d 100644 --- a/sdk/aqueduct/models/response_models.py +++ b/sdk/aqueduct/models/response_models.py @@ -227,6 +227,57 @@ class GetOperatorWithArtifactNodeResponse(BaseModel): outputs: List[uuid.UUID] +class GetOperatorWithArtifactNodeResultResponse(BaseModel): + """Represents a single merged node (metric or check) result in a workflow run. + + Attributes: + id: + The id of the operator result node. + artifact_result_id: + The id of the artifact result node. + operator_id: + The id of the operator node. + artifact_id: + The id of the artifact node. + operator_result_exec_state: + The execution state of the run operator result. + artifact_result_exec_state: + The execution state of the run artifact result. + serialization_type: + What is being serialized. + content_path: + Path to get content. + content_serialized: + If the content is too big, none. Otherwise, the content. +} + dag_id: + This id can be used to find the corresponding workflow dag version. + name: + The name of the operator. + description: + The description of the operator. + type: + The artifact type. + spec: + The operator spec. + inputs: + The id(s) of the input artifact(s) of the operator. + outputs: + The id(s) of the operator(s) that take this artifact as input. + + """ + + id: uuid.UUID + artifact_result_id: uuid.UUID + operator_id: uuid.UUID + artifact_id: uuid.UUID + operator_result_exec_state: ExecutionState + artifact_result_exec_state: ExecutionState + serialization_type: SerializationType + content_path: str + content_serialized: Optional[str] + + # V1 Responses class PreviewResponse(BaseModel): """This is the response object returned by api_client.preview(). diff --git a/src/golang/cmd/server/handler/v2/node_check_results_get.go b/src/golang/cmd/server/handler/v2/node_check_results_get.go index ae78ad5521..91bbd2becd 100644 --- a/src/golang/cmd/server/handler/v2/node_check_results_get.go +++ b/src/golang/cmd/server/handler/v2/node_check_results_get.go @@ -64,10 +64,16 @@ func (h *NodeCheckResultsGetHandler) Perform(ctx context.Context, interfaceArgs emptyResponse := []response.OperatorWithArtifactResultNode{} - dbOperatorWithArtifactNode, err := h.OperatorRepo.GetOperatorWithArtifactByArtifactIdNode(ctx, artfID, h.Database) + dbOperatorWithArtifactNodes, err := h.OperatorRepo.GetOperatorWithArtifactByArtifactIdNodeBatch(ctx, []uuid.UUID{artfID}, h.Database) if err != nil { return nil, http.StatusInternalServerError, errors.Wrap(err, "Unexpected error reading check node.") } + dbOperatorWithArtifactNode := views.OperatorWithArtifactNode{} + if len(dbOperatorWithArtifactNodes) == 0 { + return emptyResponse, http.StatusOK, nil + } else { + dbOperatorWithArtifactNode = dbOperatorWithArtifactNodes[0] + } results, err := h.OperatorResultRepo.GetOperatorWithArtifactResultNodesByOperatorNameAndWorkflow(ctx, dbOperatorWithArtifactNode.Name, wfID, h.Database) if err != nil { @@ -80,7 +86,7 @@ func (h *NodeCheckResultsGetHandler) Perform(ctx context.Context, interfaceArgs resultArtifactIds := make([]uuid.UUID, 0, len(results)) for _, result := range results { - resultArtifactIds = append(resultArtifactIds, result.ArtifactID) + resultArtifactIds = append(resultArtifactIds, result.ArtifactResultID) } artfResultToDAG, err := h.DAGRepo.GetByArtifactResultBatch(ctx, resultArtifactIds, h.Database) @@ -91,15 +97,15 @@ func (h *NodeCheckResultsGetHandler) Perform(ctx context.Context, interfaceArgs // maps from db dag Ids dbDagByDagId := make(map[uuid.UUID]models.DAG, len(artfResultToDAG)) nodeResultByDagId := make(map[uuid.UUID][]views.OperatorWithArtifactResultNode, len(artfResultToDAG)) - for _, artfResult := range results { - if dbDag, ok := artfResultToDAG[artfResult.ID]; ok { + for _, nodeResult := range results { + if dbDag, ok := artfResultToDAG[nodeResult.ArtifactResultID]; ok { if _, okDagsMap := dbDagByDagId[dbDag.ID]; !okDagsMap { dbDagByDagId[dbDag.ID] = dbDag } - nodeResultByDagId[dbDag.ID] = append(nodeResultByDagId[dbDag.ID], artfResult) + nodeResultByDagId[dbDag.ID] = append(nodeResultByDagId[dbDag.ID], nodeResult) } else { - return emptyResponse, http.StatusInternalServerError, errors.Newf("Error retrieving dag associated with artifact result %s", artfResult.ID) + return emptyResponse, http.StatusInternalServerError, errors.Newf("Error retrieving dag associated with artifact result %s", nodeResult.ArtifactResultID) } } diff --git a/src/golang/cmd/server/handler/v2/node_metric_results_get.go b/src/golang/cmd/server/handler/v2/node_metric_results_get.go index 38006779ef..930b9957ec 100644 --- a/src/golang/cmd/server/handler/v2/node_metric_results_get.go +++ b/src/golang/cmd/server/handler/v2/node_metric_results_get.go @@ -62,10 +62,16 @@ func (h *NodeMetricResultsGetHandler) Perform(ctx context.Context, interfaceArgs emptyResponse := []response.OperatorWithArtifactResultNode{} - dbOperatorWithArtifactNode, err := h.OperatorRepo.GetOperatorWithArtifactByArtifactIdNode(ctx, artfID, h.Database) + dbOperatorWithArtifactNodes, err := h.OperatorRepo.GetOperatorWithArtifactByArtifactIdNodeBatch(ctx, []uuid.UUID{artfID}, h.Database) if err != nil { return nil, http.StatusInternalServerError, errors.Wrap(err, "Unexpected error reading metric node.") } + dbOperatorWithArtifactNode := views.OperatorWithArtifactNode{} + if len(dbOperatorWithArtifactNodes) == 0 { + return emptyResponse, http.StatusOK, nil + } else { + dbOperatorWithArtifactNode = dbOperatorWithArtifactNodes[0] + } results, err := h.OperatorResultRepo.GetOperatorWithArtifactResultNodesByOperatorNameAndWorkflow(ctx, dbOperatorWithArtifactNode.Name, wfID, h.Database) if err != nil { @@ -78,7 +84,7 @@ func (h *NodeMetricResultsGetHandler) Perform(ctx context.Context, interfaceArgs resultArtifactIds := make([]uuid.UUID, 0, len(results)) for _, result := range results { - resultArtifactIds = append(resultArtifactIds, result.ArtifactID) + resultArtifactIds = append(resultArtifactIds, result.ArtifactResultID) } artfResultToDAG, err := h.DAGRepo.GetByArtifactResultBatch(ctx, resultArtifactIds, h.Database) @@ -89,15 +95,15 @@ func (h *NodeMetricResultsGetHandler) Perform(ctx context.Context, interfaceArgs // maps from db dag Ids dbDagByDagId := make(map[uuid.UUID]models.DAG, len(artfResultToDAG)) nodeResultByDagId := make(map[uuid.UUID][]views.OperatorWithArtifactResultNode, len(artfResultToDAG)) - for _, artfResult := range results { - if dbDag, ok := artfResultToDAG[artfResult.ID]; ok { + for _, NodeResult := range results { + if dbDag, ok := artfResultToDAG[NodeResult.ArtifactResultID]; ok { if _, okDagsMap := dbDagByDagId[dbDag.ID]; !okDagsMap { dbDagByDagId[dbDag.ID] = dbDag } - nodeResultByDagId[dbDag.ID] = append(nodeResultByDagId[dbDag.ID], artfResult) + nodeResultByDagId[dbDag.ID] = append(nodeResultByDagId[dbDag.ID], NodeResult) } else { - return emptyResponse, http.StatusInternalServerError, errors.Newf("Error retrieving dag associated with artifact result %s", artfResult.ID) + return emptyResponse, http.StatusInternalServerError, errors.Newf("Error retrieving dag associated with artifact result %s", NodeResult.ArtifactResultID) } } diff --git a/src/golang/lib/repos/sqlite/operator.go b/src/golang/lib/repos/sqlite/operator.go index 20a646f1cf..76e065fcd2 100644 --- a/src/golang/lib/repos/sqlite/operator.go +++ b/src/golang/lib/repos/sqlite/operator.go @@ -93,7 +93,7 @@ const operatorNodeViewSubQuery = ` WHERE op_with_outputs.outputs IS NULL ` -var mergedNodeViewSubQuery = fmt.Sprintf(` +var operatorWithArtifactNodeViewSubQuery = fmt.Sprintf(` WITH operator_node AS (%s), artifact_node AS (%s) @@ -190,7 +190,7 @@ func (*operatorReader) GetOperatorWithArtifactNodeBatch(ctx context.Context, IDs query := fmt.Sprintf( "WITH %s AS (%s) SELECT %s FROM %s WHERE %s IN (%s)", views.OperatorWithArtifactNodeView, - mergedNodeViewSubQuery, + operatorWithArtifactNodeViewSubQuery, views.OperatorWithArtifactNodeCols(), views.OperatorWithArtifactNodeView, views.OperatorWithArtifactNodeID, @@ -216,7 +216,7 @@ func (*operatorReader) GetOperatorWithArtifactByArtifactIdNodeBatch(ctx context. query := fmt.Sprintf( "WITH %s AS (%s) SELECT %s FROM %s WHERE %s IN (%s)", views.OperatorWithArtifactNodeView, - mergedNodeViewSubQuery, + operatorWithArtifactNodeViewSubQuery, views.OperatorWithArtifactNodeCols(), views.OperatorWithArtifactNodeView, views.OperatorWithArtifactNodeArtifactID, diff --git a/src/golang/lib/repos/sqlite/operator_result.go b/src/golang/lib/repos/sqlite/operator_result.go index fe3bfdef44..63ce6e67fd 100644 --- a/src/golang/lib/repos/sqlite/operator_result.go +++ b/src/golang/lib/repos/sqlite/operator_result.go @@ -204,13 +204,14 @@ func (*operatorResultReader) GetOperatorWithArtifactResultNodesByOperatorNameAnd artifact_result.id AS artifact_result_id, artifact_result.metadata, artifact_result.content_path, - artifact_result.execution_state AS artifact_result_exec_state, - FROM operator, operator_result, artifact_result, workflow_dag, workflow_dag_edge + artifact_result.execution_state AS artifact_result_exec_state + FROM operator, operator_result, artifact_result, workflow_dag, workflow_dag_edge, workflow_dag_result WHERE workflow_dag.workflow_id = $1 AND workflow_dag_edge.workflow_dag_id = workflow_dag.id + AND workflow_dag_result.workflow_dag_id = workflow_dag.id AND operator.name = $2 - AND operator_result.id = operator.id + AND operator_result.operator_id = operator.id AND workflow_dag_edge.from_id = operator.id AND workflow_dag_edge.to_id = artifact_result.artifact_id AND artifact_result.workflow_dag_result_id = workflow_dag_result.id;` From 47e6755fee3a6ca50488428a160c8d9cd434c457 Mon Sep 17 00:00:00 2001 From: Eunice Chan Date: Fri, 2 Jun 2023 14:28:15 -0700 Subject: [PATCH 4/6] Uncomment --- integration_tests/backend/test_reads.py | 1252 +++++++++++------------ 1 file changed, 626 insertions(+), 626 deletions(-) diff --git a/integration_tests/backend/test_reads.py b/integration_tests/backend/test_reads.py index b0e7084624..00132c1db8 100644 --- a/integration_tests/backend/test_reads.py +++ b/integration_tests/backend/test_reads.py @@ -104,13 +104,13 @@ def setup_class(cls): for flow_id, n_runs in cls.flows.values(): utils.wait_for_flow_runs(cls.client, flow_id, n_runs) - # @classmethod - # def teardown_class(cls): - # for flow_id, _ in cls.flows.values(): - # utils.delete_flow(cls.client, flow_id) + @classmethod + def teardown_class(cls): + for flow_id, _ in cls.flows.values(): + utils.delete_flow(cls.client, flow_id) - # for flow_id, _ in cls.running_flows.values(): - # utils.delete_flow(cls.client, flow_id) + for flow_id, _ in cls.running_flows.values(): + utils.delete_flow(cls.client, flow_id) @classmethod def response(cls, endpoint, additional_headers): @@ -131,558 +131,558 @@ def post_response(cls, endpoint, additional_headers={}): r = requests.post(url, headers=headers) return r - # def test_endpoint_list_workflow_tables(self): - # endpoint = self.LIST_WORKFLOW_SAVED_OBJECTS_TEMPLATE % self.flows["changing_saves"][0] - # data = self.get_response(endpoint).json()["object_details"] - - # assert len(data) == 3 - - # # table_name, update_mode - # data_set = set( - # [ - # ("table_1", "append"), - # ("table_1", "replace"), - # ("table_2", "replace"), - # ] - # ) - - # assert ( - # set( - # [ - # (item["spec"]["parameters"]["table"], item["spec"]["parameters"]["update_mode"]) - # for item in data - # ] - # ) - # == data_set - # ) - - # # Check all in same resource - # assert len(set([item["resource_name"] for item in data])) == 1 - # assert len(set([item["spec"]["service"] for item in data])) == 1 - - # def test_endpoint_delete_resource(self): - # resource_name = f"test_delete_resource_{uuid.uuid4().hex[:8]}" - - # # Check resource did not exist - # data = self.get_response(self.LIST_INTEGRATIONS_TEMPLATE).json() - # assert resource_name not in set([resource["name"] for resource in data]) - - # # Create resource - # status = self.post_response( - # self.CONNECT_INTEGRATION_TEMPLATE, - # additional_headers={ - # "resource-name": resource_name, - # "resource-service": "SQLite", - # "resource-config": json.dumps({"database": self.DEMO_DB_PATH}), - # }, - # ).status_code - # assert status == 200 - - # # Check resource created - # data = self.get_response(self.LIST_INTEGRATIONS_TEMPLATE).json() - # resource_data = {resource["name"]: resource["id"] for resource in data} - # assert resource_name in set(resource_data.keys()) - - # # Delete resource - # status = self.post_response( - # self.DELETE_INTEGRATION_TEMPLATE % resource_data[resource_name] - # ).status_code - # assert status == 200 - - # # Check resource does not exist - # data = self.get_response(self.LIST_INTEGRATIONS_TEMPLATE).json() - # assert resource_name not in set([resource["name"] for resource in data]) - - # def test_endpoint_test_resource(self): - # resp = self.get_response(self.GET_TEST_INTEGRATION_TEMPLATE % self.resource.id()) - # assert resp.ok - - # def test_endpoint_get_workflow_dag_result_with_failure(self): - # flow_id = self.flows["flow_with_failure"][0] - # flow = self.client.flow(flow_id) - # runs = flow.list_runs() - # resp = self.get_response( - # self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) - # ).json() - # assert_exec_state(resp["result"]["exec_state"], "failed") - # # operators - # operators = resp["operators"] - # assert len(operators) == 3 - # for op in operators.values(): - # name = op["name"] - # exec_state = op["result"]["exec_state"] - - # if "query" in name: # extract - # assert_exec_state(exec_state, "succeeded") - # elif name == "bad_op": - # assert_exec_state(exec_state, "failed") - # elif name == "bad_op_downstream": - # assert_exec_state(exec_state, "canceled") - # else: - # raise Exception(f"unexpected operator name {name}") - - # # artifacts - # artifacts = resp["artifacts"] - # assert len(artifacts) == 3 - # for artf in artifacts.values(): - # name = artf["name"] - # exec_state = artf["result"]["exec_state"] - - # if "query" in name: - # assert_exec_state(exec_state, "succeeded") - # elif name == "bad_op artifact": - # assert_exec_state(exec_state, "canceled") - # elif name == "bad_op_downstream artifact": - # assert_exec_state(exec_state, "canceled") - # else: - # raise Exception(f"unexpected operator name {name}") - - # def test_endpoint_get_workflow_dag_result_with_metrics_and_checks(self): - # flow_id = self.flows["flow_with_metrics_and_checks"][0] - # flow = self.client.flow(flow_id) - # runs = flow.list_runs() - # resp = self.get_response( - # self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) - # ).json() - # assert_exec_state(resp["result"]["exec_state"], "succeeded") - - # # operators - # operators = resp["operators"] - # assert len(operators) == 3 - # for op in operators.values(): - # name = op["name"] - # exec_state = op["result"]["exec_state"] - # if "query" in name or name == "size" or name == "check": # extract - # assert_exec_state(exec_state, "succeeded") - # else: - # raise Exception(f"unexpected operator name {name}") - - # # artifacts - # artifacts = resp["artifacts"] - # assert len(artifacts) == 3 - # for artf in artifacts.values(): - # name = artf["name"] - # exec_state = artf["result"]["exec_state"] - # value = artf["result"]["content_serialized"] - - # if "query" in name: - # assert_exec_state(exec_state, "succeeded") - # elif name == "size artifact": - # assert_exec_state(exec_state, "succeeded") - # assert int(value) > 0 - # elif name == "check artifact": - # assert_exec_state(exec_state, "succeeded") - # assert value == "true" - # else: - # raise Exception(f"unexpected operator name {name}") - - # def test_endpoint_get_workflow_dag_result_on_flow_with_sleep(self): - # flow_id = self.running_flows["flow_with_sleep"][0] - # flow = self.client.flow(flow_id) - # runs = flow.list_runs() - # resp = self.get_response( - # self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) - # ).json() - # assert_exec_state(resp["result"]["exec_state"], "pending") - - # # operators - # operators = resp["operators"] - # assert len(operators) == 2 - # for op in operators.values(): - # name = op["name"] - # exec_state = op["result"]["exec_state"] - # if "query" in name: # extract - # assert_exec_state(exec_state, "succeeded") - # elif name == "sleeping_op": - # assert_exec_state(exec_state, "pending") - # else: - # raise Exception(f"unexpected operator name {name}") - - # # artifacts - # artifacts = resp["artifacts"] - # assert len(artifacts) == 2 - # for artf in artifacts.values(): - # name = artf["name"] - # exec_state = artf["result"]["exec_state"] - - # if "query" in name: - # assert_exec_state(exec_state, "succeeded") - # elif name == "sleeping_op artifact": - # assert_exec_state(exec_state, "pending") - # else: - # raise Exception(f"unexpected operator name {name}") - - # def test_endpoint_list_artifact_results_with_metrics_and_checks(self): - # flow_id, num_runs = self.flows["flow_with_metrics_and_checks"] - # flow = self.client.flow(flow_id) - # runs = flow.list_runs() - # resp = self.get_response( - # self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) - # ).json() - - # # artifacts - # artifacts = resp["artifacts"] - # assert len(artifacts) == 3 - # for artf in artifacts.values(): - # name = artf["name"] - # id = artf["id"] - # resp = self.get_response(self.LIST_ARTIFACT_RESULTS_TEMPLATE % (flow_id, id)).json() - # results = resp["results"] - # assert len(results) == num_runs - - # for result in results: - # exec_state = result["exec_state"] - # value = result["content_serialized"] - # assert_exec_state(exec_state, "succeeded") - - # if "query" in name: - # assert value is None - # elif name == "size artifact": - # assert int(value) > 0 - # elif name == "check artifact": - # assert value == "true" - - # def test_endpoint_workflows_get(self): - # resp = self.get_response(self.GET_WORKFLOWS_TEMPLATE) - # resp = resp.json() - - # if len(resp) > 0: - # keys = [ - # "id", - # "user_id", - # "name", - # "description", - # "schedule", - # "created_at", - # "retention_policy", - # "notification_settings", - # ] - - # user_id = resp[0]["user_id"] - - # for v2_workflow in resp: - # for key in keys: - # assert key in v2_workflow - # assert v2_workflow["user_id"] == user_id - - # def test_endpoint_workflow_dags_get(self): - # flow_id, _ = self.flows["flow_with_metrics_and_checks"] - # resp = self.get_response(self.GET_DAGS_TEMPLATE % flow_id) - # resp = resp.json() - - # assert len(resp) == 2 - # for dag_dict in resp: - # dag = GetDagResponse(**dag_dict) - # assert dag.workflow_id == flow_id - # assert dag.created_at != "" - # assert dag.engine_config.type == RuntimeType.AQUEDUCT - - # def test_endpoint_dag_results_get(self): - # flow_id, n_runs = self.flows["flow_with_metrics_and_checks"] - # resp = self.get_response(self.GET_DAG_RESULTS_TEMPLATE % flow_id).json() - - # assert len(resp) == n_runs - - # def check_structure(resp, all_succeeded=False): - # for result in resp: - # result = GetDagResultResponse(**result) - # if all_succeeded: - # assert result.exec_state.status == "succeeded" - # assert result.exec_state.failure_type == None - # assert result.exec_state.error == None - - # check_structure(resp, all_succeeded=True) - - # # Using the order parameter - # flow_id, n_runs = self.flows["flow_with_failure"] - # resp = self.get_response( - # self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status", - # ).json() - - # check_structure(resp) - # statuses = [result["exec_state"]["status"] for result in resp] - # sorted_statuses = sorted(statuses, reverse=True) # Descending order - # assert statuses == sorted_statuses - - # # Default is descending - # flow_id, n_runs = self.flows["flow_with_failure"] - # resp = self.get_response( - # self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status&order_descending=true", - # ).json() - - # check_structure(resp) - # descending_statuses = [result["exec_state"]["status"] for result in resp] - # assert statuses == descending_statuses - - # # Ascending works - # flow_id, n_runs = self.flows["flow_with_failure"] - # resp = self.get_response( - # self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status&order_descending=false", - # ).json() - - # check_structure(resp) - # ascending_statuses = [result["exec_state"]["status"] for result in resp] - # assert descending_statuses[::-1] == ascending_statuses - - # # Using the limit parameter - # resp = self.get_response( - # self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?limit=1", - # ).json() - - # check_structure(resp) - # assert len(resp) == 1 - - # # Using both the order and limit parameters - # resp = self.get_response( - # self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status&limit=1", - # ).json() - - # check_structure(resp) - # workflow_status = [result["exec_state"]["status"] for result in resp] - # assert len(workflow_status) == 1 - # workflow_status = workflow_status[0] - # assert workflow_status == sorted_statuses[0] - - # def test_endpoint_nodes_get(self): - # for flow_id, _ in [ - # self.flows["flow_with_metrics_and_checks"], - # self.flows["flow_with_multiple_operators"], - # ]: - # flow = self.client.flow(flow_id) - # workflow_resp = flow._get_workflow_resp() - # dag_id = list(workflow_resp.workflow_dags.keys())[0] - # resp = self.get_response(self.GET_NODES_TEMPLATE % (flow_id, dag_id)).json() - - # all_output_counts = [] - # for operator in resp["operators"]: - # result = GetNodeOperatorResponse(**operator) - # all_output_counts.append(len(result.outputs)) - # assert sum(all_output_counts) == len(all_output_counts) - # assert set(all_output_counts) == set([1]) - - # all_output_counts = [] - # for artifact in resp["artifacts"]: - # result = GetNodeArtifactResponse(**artifact) - # all_output_counts.append(len(result.outputs)) - # assert sum(all_output_counts) == len(all_output_counts) - 1 - # assert set(all_output_counts) == set([0, 1]) - - # def test_endpoint_nodes_results_get(self): - # for flow_id, _ in [ - # self.flows["flow_with_metrics_and_checks"], - # self.flows["flow_with_multiple_operators"], - # ]: - # flow = self.client.flow(flow_id) - # workflow_resp = flow._get_workflow_resp() - # dag_result_id = workflow_resp.workflow_dag_results[0].id - # resp = self.get_response( - # self.GET_NODES_RESULTS_TEMPLATE % (flow_id, dag_result_id) - # ).json() - # assert "operators" in resp.keys() - # assert "artifacts" in resp.keys() - # assert len(resp["operators"]) == len(resp["artifacts"]) - # for op in resp["operators"]: - # result = GetOperatorResultResponse(**op) - # result.exec_state.status == "succeeded" - # for artf in resp["artifacts"]: - # result = GetArtifactResultResponse(**artf) - # result.exec_state.status == "succeeded" - - # def test_endpoint_node_artifact_get(self): - # for flow_id, _ in [ - # self.flows["flow_with_metrics_and_checks"], - # self.flows["flow_with_multiple_operators"], - # ]: - # flow = self.client.flow(flow_id) - # workflow_resp = flow._get_workflow_resp() - # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - # dag_result_id = workflow_resp.workflow_dag_results[0].id - - # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - # flow_id, - # dag_result_id, - # ) - # artifact_ids = list(dag_result_resp.artifacts.keys()) - # artifact_id = str(artifact_ids[0]) - # all_output_counts = [] - # for artifact_id in artifact_ids: - # artifact_id = str(artifact_id) - # resp = self.get_response( - # self.GET_NODE_ARTIFACT_TEMPLATE % (flow_id, dag_id, artifact_id) - # ).json() - # result = GetNodeArtifactResponse(**resp) - # all_output_counts.append(len(result.outputs)) - # assert sum(all_output_counts) == len(all_output_counts) - 1 - # assert set(all_output_counts) == set([0, 1]) - - # def test_endpoint_node_artifact_result_content_get(self): - # for flow_id, _ in [ - # self.flows["flow_with_metrics_and_checks"], - # self.flows["flow_with_multiple_operators"], - # ]: - # flow = self.client.flow(flow_id) - # workflow_resp = flow._get_workflow_resp() - # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - # dag_result_id = workflow_resp.workflow_dag_results[0].id - - # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - # flow_id, - # dag_result_id, - # ) - # artifact_ids = list(dag_result_resp.artifacts.keys()) - # artifact_id = str(artifact_ids[0]) - - # resp = self.get_response( - # self.GET_NODE_ARTIFACT_RESULTS_TEMPLATE % (flow_id, dag_id, artifact_id) - # ).json() - # downstream_ids = [GetArtifactResultResponse(**result).id for result in resp] - # for downstream_id in downstream_ids: - # artifact_result_id = str(downstream_id) - # resp = self.get_response( - # self.GET_NODE_ARTIFACT_RESULT_CONTENT_TEMPLATE - # % (flow_id, dag_id, artifact_id, artifact_result_id) - # ) - # assert resp.ok - # resp_obj = GetNodeResultContentResponse(**resp.json()) - # # One of these should be successful (direct descendent of operator) - # assert not resp_obj.is_downsampled - # assert len(resp_obj.content) > 0 - - # def test_endpoint_node_artifact_results_get(self): - # for flow_id, _ in [ - # self.flows["flow_with_metrics_and_checks"], - # self.flows["flow_with_multiple_operators"], - # ]: - # flow = self.client.flow(flow_id) - # workflow_resp = flow._get_workflow_resp() - # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - # dag_result_id = workflow_resp.workflow_dag_results[0].id - - # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - # flow_id, - # dag_result_id, - # ) - # artifact_ids = list(dag_result_resp.artifacts.keys()) - # artifact_id = str(artifact_ids[0]) - - # resp = self.get_response( - # self.GET_NODE_ARTIFACT_RESULTS_TEMPLATE % (flow_id, dag_id, artifact_id) - # ).json() - # for result in resp: - # result = GetArtifactResultResponse(**result) - - # def test_endpoint_node_operator_get(self): - # for flow_id, _ in [ - # self.flows["flow_with_metrics_and_checks"], - # self.flows["flow_with_multiple_operators"], - # ]: - # flow = self.client.flow(flow_id) - # workflow_resp = flow._get_workflow_resp() - # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - # dag_result_id = workflow_resp.workflow_dag_results[0].id - - # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - # flow_id, - # dag_result_id, - # ) - # operator_ids = list(dag_result_resp.operators.keys()) - # operator_id = str(operator_ids[0]) - - # resp = self.get_response( - # self.GET_NODE_OPERATOR_TEMPLATE % (flow_id, dag_id, operator_id) - # ).json() - # result = GetNodeOperatorResponse(**resp) - # assert str(result.id) == operator_id - # assert result.dag_id == dag_id - - # def test_endpoint_node_operator_content_get(self): - # flow_id, _ = self.flows["flow_with_multiple_operators"] - # flow = self.client.flow(flow_id) - # workflow_resp = flow._get_workflow_resp() - # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - # dag_result_id = workflow_resp.workflow_dag_results[0].id - - # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - # flow_id, - # dag_result_id, - # ) - # operator_ids = list(dag_result_resp.operators.keys()) - # operator_id = str(operator_ids[0]) - - # resp = self.get_response( - # self.GET_NODE_OPERATOR_CONTENT_TEMPLATE % (flow_id, dag_id, operator_id) - # ) - # # The response is a form data. For now, we simply check the response's code. - # assert resp.ok - - # def test_endpoint_node_metric_get(self): - # flow_id, _ = self.flows["flow_with_metrics_and_checks"] - # flow = self.client.flow(flow_id) - # workflow_resp = flow._get_workflow_resp() - # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - # dag_result_id = workflow_resp.workflow_dag_results[0].id - - # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - # flow_id, - # dag_result_id, - # ) - # operator_ids = [ - # id - # for id in dag_result_resp.operators.keys() - # if dag_result_resp.operators[id].spec.metric - # ] - # operator_id = str(operator_ids[0]) - - # resp = self.get_response( - # self.GET_NODE_METRIC_TEMPLATE % (flow_id, dag_id, operator_id) - # ).json() - # result = GetOperatorWithArtifactNodeResponse(**resp) - # assert str(result.id) == operator_id - # assert result.dag_id == dag_id - # assert len(result.inputs) == 1 - # assert len(result.outputs) == 1 - - # def test_endpoint_node_metric_result_content_get(self): - # flow_id, _ = self.flows["flow_with_metrics_and_checks"] - # flow = self.client.flow(flow_id) - # workflow_resp = flow._get_workflow_resp() - # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - # dag_result_id = workflow_resp.workflow_dag_results[0].id - - # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - # flow_id, - # dag_result_id, - # ) - # operator_ids = [ - # id - # for id in dag_result_resp.operators.keys() - # if dag_result_resp.operators[id].spec.metric - # ] - # operator_id = str(operator_ids[0]) - - # resp = self.get_response( - # self.GET_NODE_METRIC_TEMPLATE % (flow_id, dag_id, operator_id) - # ).json() - - # result = GetOperatorWithArtifactNodeResponse(**resp) - - # artifact_id = result.artifact_id - - # resp = self.get_response( - # self.LIST_ARTIFACT_RESULTS_TEMPLATE % (flow_id, artifact_id) - # ).json() - # results = resp["results"] - # # One of these should be correct for the DAG run and can get result content. - # for artifact_result in results: - # resp = self.get_response( - # self.GET_NODE_METRIC_RESULT_CONTENT_TEMPLATE - # % (flow_id, dag_id, operator_id, artifact_result["id"]) - # ) - # assert resp.ok - # resp_obj = GetNodeResultContentResponse(**resp.json()) - # # One of these should be successful (direct descendent of operator) - # assert not resp_obj.is_downsampled - # assert len(resp_obj.content) > 0 + def test_endpoint_list_workflow_tables(self): + endpoint = self.LIST_WORKFLOW_SAVED_OBJECTS_TEMPLATE % self.flows["changing_saves"][0] + data = self.get_response(endpoint).json()["object_details"] + + assert len(data) == 3 + + # table_name, update_mode + data_set = set( + [ + ("table_1", "append"), + ("table_1", "replace"), + ("table_2", "replace"), + ] + ) + + assert ( + set( + [ + (item["spec"]["parameters"]["table"], item["spec"]["parameters"]["update_mode"]) + for item in data + ] + ) + == data_set + ) + + # Check all in same resource + assert len(set([item["resource_name"] for item in data])) == 1 + assert len(set([item["spec"]["service"] for item in data])) == 1 + + def test_endpoint_delete_resource(self): + resource_name = f"test_delete_resource_{uuid.uuid4().hex[:8]}" + + # Check resource did not exist + data = self.get_response(self.LIST_INTEGRATIONS_TEMPLATE).json() + assert resource_name not in set([resource["name"] for resource in data]) + + # Create resource + status = self.post_response( + self.CONNECT_INTEGRATION_TEMPLATE, + additional_headers={ + "resource-name": resource_name, + "resource-service": "SQLite", + "resource-config": json.dumps({"database": self.DEMO_DB_PATH}), + }, + ).status_code + assert status == 200 + + # Check resource created + data = self.get_response(self.LIST_INTEGRATIONS_TEMPLATE).json() + resource_data = {resource["name"]: resource["id"] for resource in data} + assert resource_name in set(resource_data.keys()) + + # Delete resource + status = self.post_response( + self.DELETE_INTEGRATION_TEMPLATE % resource_data[resource_name] + ).status_code + assert status == 200 + + # Check resource does not exist + data = self.get_response(self.LIST_INTEGRATIONS_TEMPLATE).json() + assert resource_name not in set([resource["name"] for resource in data]) + + def test_endpoint_test_resource(self): + resp = self.get_response(self.GET_TEST_INTEGRATION_TEMPLATE % self.resource.id()) + assert resp.ok + + def test_endpoint_get_workflow_dag_result_with_failure(self): + flow_id = self.flows["flow_with_failure"][0] + flow = self.client.flow(flow_id) + runs = flow.list_runs() + resp = self.get_response( + self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) + ).json() + assert_exec_state(resp["result"]["exec_state"], "failed") + # operators + operators = resp["operators"] + assert len(operators) == 3 + for op in operators.values(): + name = op["name"] + exec_state = op["result"]["exec_state"] + + if "query" in name: # extract + assert_exec_state(exec_state, "succeeded") + elif name == "bad_op": + assert_exec_state(exec_state, "failed") + elif name == "bad_op_downstream": + assert_exec_state(exec_state, "canceled") + else: + raise Exception(f"unexpected operator name {name}") + + # artifacts + artifacts = resp["artifacts"] + assert len(artifacts) == 3 + for artf in artifacts.values(): + name = artf["name"] + exec_state = artf["result"]["exec_state"] + + if "query" in name: + assert_exec_state(exec_state, "succeeded") + elif name == "bad_op artifact": + assert_exec_state(exec_state, "canceled") + elif name == "bad_op_downstream artifact": + assert_exec_state(exec_state, "canceled") + else: + raise Exception(f"unexpected operator name {name}") + + def test_endpoint_get_workflow_dag_result_with_metrics_and_checks(self): + flow_id = self.flows["flow_with_metrics_and_checks"][0] + flow = self.client.flow(flow_id) + runs = flow.list_runs() + resp = self.get_response( + self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) + ).json() + assert_exec_state(resp["result"]["exec_state"], "succeeded") + + # operators + operators = resp["operators"] + assert len(operators) == 3 + for op in operators.values(): + name = op["name"] + exec_state = op["result"]["exec_state"] + if "query" in name or name == "size" or name == "check": # extract + assert_exec_state(exec_state, "succeeded") + else: + raise Exception(f"unexpected operator name {name}") + + # artifacts + artifacts = resp["artifacts"] + assert len(artifacts) == 3 + for artf in artifacts.values(): + name = artf["name"] + exec_state = artf["result"]["exec_state"] + value = artf["result"]["content_serialized"] + + if "query" in name: + assert_exec_state(exec_state, "succeeded") + elif name == "size artifact": + assert_exec_state(exec_state, "succeeded") + assert int(value) > 0 + elif name == "check artifact": + assert_exec_state(exec_state, "succeeded") + assert value == "true" + else: + raise Exception(f"unexpected operator name {name}") + + def test_endpoint_get_workflow_dag_result_on_flow_with_sleep(self): + flow_id = self.running_flows["flow_with_sleep"][0] + flow = self.client.flow(flow_id) + runs = flow.list_runs() + resp = self.get_response( + self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) + ).json() + assert_exec_state(resp["result"]["exec_state"], "pending") + + # operators + operators = resp["operators"] + assert len(operators) == 2 + for op in operators.values(): + name = op["name"] + exec_state = op["result"]["exec_state"] + if "query" in name: # extract + assert_exec_state(exec_state, "succeeded") + elif name == "sleeping_op": + assert_exec_state(exec_state, "pending") + else: + raise Exception(f"unexpected operator name {name}") + + # artifacts + artifacts = resp["artifacts"] + assert len(artifacts) == 2 + for artf in artifacts.values(): + name = artf["name"] + exec_state = artf["result"]["exec_state"] + + if "query" in name: + assert_exec_state(exec_state, "succeeded") + elif name == "sleeping_op artifact": + assert_exec_state(exec_state, "pending") + else: + raise Exception(f"unexpected operator name {name}") + + def test_endpoint_list_artifact_results_with_metrics_and_checks(self): + flow_id, num_runs = self.flows["flow_with_metrics_and_checks"] + flow = self.client.flow(flow_id) + runs = flow.list_runs() + resp = self.get_response( + self.GET_WORKFLOW_RESULT_TEMPLATE % (flow_id, runs[0]["run_id"]) + ).json() + + # artifacts + artifacts = resp["artifacts"] + assert len(artifacts) == 3 + for artf in artifacts.values(): + name = artf["name"] + id = artf["id"] + resp = self.get_response(self.LIST_ARTIFACT_RESULTS_TEMPLATE % (flow_id, id)).json() + results = resp["results"] + assert len(results) == num_runs + + for result in results: + exec_state = result["exec_state"] + value = result["content_serialized"] + assert_exec_state(exec_state, "succeeded") + + if "query" in name: + assert value is None + elif name == "size artifact": + assert int(value) > 0 + elif name == "check artifact": + assert value == "true" + + def test_endpoint_workflows_get(self): + resp = self.get_response(self.GET_WORKFLOWS_TEMPLATE) + resp = resp.json() + + if len(resp) > 0: + keys = [ + "id", + "user_id", + "name", + "description", + "schedule", + "created_at", + "retention_policy", + "notification_settings", + ] + + user_id = resp[0]["user_id"] + + for v2_workflow in resp: + for key in keys: + assert key in v2_workflow + assert v2_workflow["user_id"] == user_id + + def test_endpoint_workflow_dags_get(self): + flow_id, _ = self.flows["flow_with_metrics_and_checks"] + resp = self.get_response(self.GET_DAGS_TEMPLATE % flow_id) + resp = resp.json() + + assert len(resp) == 2 + for dag_dict in resp: + dag = GetDagResponse(**dag_dict) + assert dag.workflow_id == flow_id + assert dag.created_at != "" + assert dag.engine_config.type == RuntimeType.AQUEDUCT + + def test_endpoint_dag_results_get(self): + flow_id, n_runs = self.flows["flow_with_metrics_and_checks"] + resp = self.get_response(self.GET_DAG_RESULTS_TEMPLATE % flow_id).json() + + assert len(resp) == n_runs + + def check_structure(resp, all_succeeded=False): + for result in resp: + result = GetDagResultResponse(**result) + if all_succeeded: + assert result.exec_state.status == "succeeded" + assert result.exec_state.failure_type == None + assert result.exec_state.error == None + + check_structure(resp, all_succeeded=True) + + # Using the order parameter + flow_id, n_runs = self.flows["flow_with_failure"] + resp = self.get_response( + self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status", + ).json() + + check_structure(resp) + statuses = [result["exec_state"]["status"] for result in resp] + sorted_statuses = sorted(statuses, reverse=True) # Descending order + assert statuses == sorted_statuses + + # Default is descending + flow_id, n_runs = self.flows["flow_with_failure"] + resp = self.get_response( + self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status&order_descending=true", + ).json() + + check_structure(resp) + descending_statuses = [result["exec_state"]["status"] for result in resp] + assert statuses == descending_statuses + + # Ascending works + flow_id, n_runs = self.flows["flow_with_failure"] + resp = self.get_response( + self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status&order_descending=false", + ).json() + + check_structure(resp) + ascending_statuses = [result["exec_state"]["status"] for result in resp] + assert descending_statuses[::-1] == ascending_statuses + + # Using the limit parameter + resp = self.get_response( + self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?limit=1", + ).json() + + check_structure(resp) + assert len(resp) == 1 + + # Using both the order and limit parameters + resp = self.get_response( + self.GET_DAG_RESULTS_TEMPLATE % flow_id + "?order_by=status&limit=1", + ).json() + + check_structure(resp) + workflow_status = [result["exec_state"]["status"] for result in resp] + assert len(workflow_status) == 1 + workflow_status = workflow_status[0] + assert workflow_status == sorted_statuses[0] + + def test_endpoint_nodes_get(self): + for flow_id, _ in [ + self.flows["flow_with_metrics_and_checks"], + self.flows["flow_with_multiple_operators"], + ]: + flow = self.client.flow(flow_id) + workflow_resp = flow._get_workflow_resp() + dag_id = list(workflow_resp.workflow_dags.keys())[0] + resp = self.get_response(self.GET_NODES_TEMPLATE % (flow_id, dag_id)).json() + + all_output_counts = [] + for operator in resp["operators"]: + result = GetNodeOperatorResponse(**operator) + all_output_counts.append(len(result.outputs)) + assert sum(all_output_counts) == len(all_output_counts) + assert set(all_output_counts) == set([1]) + + all_output_counts = [] + for artifact in resp["artifacts"]: + result = GetNodeArtifactResponse(**artifact) + all_output_counts.append(len(result.outputs)) + assert sum(all_output_counts) == len(all_output_counts) - 1 + assert set(all_output_counts) == set([0, 1]) + + def test_endpoint_nodes_results_get(self): + for flow_id, _ in [ + self.flows["flow_with_metrics_and_checks"], + self.flows["flow_with_multiple_operators"], + ]: + flow = self.client.flow(flow_id) + workflow_resp = flow._get_workflow_resp() + dag_result_id = workflow_resp.workflow_dag_results[0].id + resp = self.get_response( + self.GET_NODES_RESULTS_TEMPLATE % (flow_id, dag_result_id) + ).json() + assert "operators" in resp.keys() + assert "artifacts" in resp.keys() + assert len(resp["operators"]) == len(resp["artifacts"]) + for op in resp["operators"]: + result = GetOperatorResultResponse(**op) + result.exec_state.status == "succeeded" + for artf in resp["artifacts"]: + result = GetArtifactResultResponse(**artf) + result.exec_state.status == "succeeded" + + def test_endpoint_node_artifact_get(self): + for flow_id, _ in [ + self.flows["flow_with_metrics_and_checks"], + self.flows["flow_with_multiple_operators"], + ]: + flow = self.client.flow(flow_id) + workflow_resp = flow._get_workflow_resp() + dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + dag_result_id = workflow_resp.workflow_dag_results[0].id + + dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + flow_id, + dag_result_id, + ) + artifact_ids = list(dag_result_resp.artifacts.keys()) + artifact_id = str(artifact_ids[0]) + all_output_counts = [] + for artifact_id in artifact_ids: + artifact_id = str(artifact_id) + resp = self.get_response( + self.GET_NODE_ARTIFACT_TEMPLATE % (flow_id, dag_id, artifact_id) + ).json() + result = GetNodeArtifactResponse(**resp) + all_output_counts.append(len(result.outputs)) + assert sum(all_output_counts) == len(all_output_counts) - 1 + assert set(all_output_counts) == set([0, 1]) + + def test_endpoint_node_artifact_result_content_get(self): + for flow_id, _ in [ + self.flows["flow_with_metrics_and_checks"], + self.flows["flow_with_multiple_operators"], + ]: + flow = self.client.flow(flow_id) + workflow_resp = flow._get_workflow_resp() + dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + dag_result_id = workflow_resp.workflow_dag_results[0].id + + dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + flow_id, + dag_result_id, + ) + artifact_ids = list(dag_result_resp.artifacts.keys()) + artifact_id = str(artifact_ids[0]) + + resp = self.get_response( + self.GET_NODE_ARTIFACT_RESULTS_TEMPLATE % (flow_id, dag_id, artifact_id) + ).json() + downstream_ids = [GetArtifactResultResponse(**result).id for result in resp] + for downstream_id in downstream_ids: + artifact_result_id = str(downstream_id) + resp = self.get_response( + self.GET_NODE_ARTIFACT_RESULT_CONTENT_TEMPLATE + % (flow_id, dag_id, artifact_id, artifact_result_id) + ) + assert resp.ok + resp_obj = GetNodeResultContentResponse(**resp.json()) + # One of these should be successful (direct descendent of operator) + assert not resp_obj.is_downsampled + assert len(resp_obj.content) > 0 + + def test_endpoint_node_artifact_results_get(self): + for flow_id, _ in [ + self.flows["flow_with_metrics_and_checks"], + self.flows["flow_with_multiple_operators"], + ]: + flow = self.client.flow(flow_id) + workflow_resp = flow._get_workflow_resp() + dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + dag_result_id = workflow_resp.workflow_dag_results[0].id + + dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + flow_id, + dag_result_id, + ) + artifact_ids = list(dag_result_resp.artifacts.keys()) + artifact_id = str(artifact_ids[0]) + + resp = self.get_response( + self.GET_NODE_ARTIFACT_RESULTS_TEMPLATE % (flow_id, dag_id, artifact_id) + ).json() + for result in resp: + result = GetArtifactResultResponse(**result) + + def test_endpoint_node_operator_get(self): + for flow_id, _ in [ + self.flows["flow_with_metrics_and_checks"], + self.flows["flow_with_multiple_operators"], + ]: + flow = self.client.flow(flow_id) + workflow_resp = flow._get_workflow_resp() + dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + dag_result_id = workflow_resp.workflow_dag_results[0].id + + dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + flow_id, + dag_result_id, + ) + operator_ids = list(dag_result_resp.operators.keys()) + operator_id = str(operator_ids[0]) + + resp = self.get_response( + self.GET_NODE_OPERATOR_TEMPLATE % (flow_id, dag_id, operator_id) + ).json() + result = GetNodeOperatorResponse(**resp) + assert str(result.id) == operator_id + assert result.dag_id == dag_id + + def test_endpoint_node_operator_content_get(self): + flow_id, _ = self.flows["flow_with_multiple_operators"] + flow = self.client.flow(flow_id) + workflow_resp = flow._get_workflow_resp() + dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + dag_result_id = workflow_resp.workflow_dag_results[0].id + + dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + flow_id, + dag_result_id, + ) + operator_ids = list(dag_result_resp.operators.keys()) + operator_id = str(operator_ids[0]) + + resp = self.get_response( + self.GET_NODE_OPERATOR_CONTENT_TEMPLATE % (flow_id, dag_id, operator_id) + ) + # The response is a form data. For now, we simply check the response's code. + assert resp.ok + + def test_endpoint_node_metric_get(self): + flow_id, _ = self.flows["flow_with_metrics_and_checks"] + flow = self.client.flow(flow_id) + workflow_resp = flow._get_workflow_resp() + dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + dag_result_id = workflow_resp.workflow_dag_results[0].id + + dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + flow_id, + dag_result_id, + ) + operator_ids = [ + id + for id in dag_result_resp.operators.keys() + if dag_result_resp.operators[id].spec.metric + ] + operator_id = str(operator_ids[0]) + + resp = self.get_response( + self.GET_NODE_METRIC_TEMPLATE % (flow_id, dag_id, operator_id) + ).json() + result = GetOperatorWithArtifactNodeResponse(**resp) + assert str(result.id) == operator_id + assert result.dag_id == dag_id + assert len(result.inputs) == 1 + assert len(result.outputs) == 1 + + def test_endpoint_node_metric_result_content_get(self): + flow_id, _ = self.flows["flow_with_metrics_and_checks"] + flow = self.client.flow(flow_id) + workflow_resp = flow._get_workflow_resp() + dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + dag_result_id = workflow_resp.workflow_dag_results[0].id + + dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + flow_id, + dag_result_id, + ) + operator_ids = [ + id + for id in dag_result_resp.operators.keys() + if dag_result_resp.operators[id].spec.metric + ] + operator_id = str(operator_ids[0]) + + resp = self.get_response( + self.GET_NODE_METRIC_TEMPLATE % (flow_id, dag_id, operator_id) + ).json() + + result = GetOperatorWithArtifactNodeResponse(**resp) + + artifact_id = result.artifact_id + + resp = self.get_response( + self.LIST_ARTIFACT_RESULTS_TEMPLATE % (flow_id, artifact_id) + ).json() + results = resp["results"] + # One of these should be correct for the DAG run and can get result content. + for artifact_result in results: + resp = self.get_response( + self.GET_NODE_METRIC_RESULT_CONTENT_TEMPLATE + % (flow_id, dag_id, operator_id, artifact_result["id"]) + ) + assert resp.ok + resp_obj = GetNodeResultContentResponse(**resp.json()) + # One of these should be successful (direct descendent of operator) + assert not resp_obj.is_downsampled + assert len(resp_obj.content) > 0 def test_endpoint_node_metric_results_get(self): flow_id = self.flows["flow_with_metrics_and_checks"][0] @@ -706,74 +706,74 @@ def test_endpoint_node_metric_results_get(self): for result in resp: result = GetOperatorWithArtifactNodeResultResponse(**result) - # def test_endpoint_node_check_get(self): - # flow_id, _ = self.flows["flow_with_metrics_and_checks"] - # flow = self.client.flow(flow_id) - # workflow_resp = flow._get_workflow_resp() - # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - # dag_result_id = workflow_resp.workflow_dag_results[0].id - - # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - # flow_id, - # dag_result_id, - # ) - # operator_ids = [ - # id - # for id in dag_result_resp.operators.keys() - # if dag_result_resp.operators[id].spec.check - # ] - # operator_id = str(operator_ids[0]) - - # resp = self.get_response( - # self.GET_NODE_CHECK_TEMPLATE % (flow_id, dag_id, operator_id) - # ).json() - # result = GetOperatorWithArtifactNodeResponse(**resp) - # assert str(result.id) == operator_id - # assert result.dag_id == dag_id - # assert len(result.inputs) == 1 - # assert len(result.outputs) == 0 - - # def test_endpoint_node_check_result_content_get(self): - # flow_id, _ = self.flows["flow_with_metrics_and_checks"] - # flow = self.client.flow(flow_id) - # workflow_resp = flow._get_workflow_resp() - # dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id - # dag_result_id = workflow_resp.workflow_dag_results[0].id - - # dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( - # flow_id, - # dag_result_id, - # ) - # operator_ids = [ - # id - # for id in dag_result_resp.operators.keys() - # if dag_result_resp.operators[id].spec.check - # ] - # operator_id = str(operator_ids[0]) - - # resp = self.get_response( - # self.GET_NODE_CHECK_TEMPLATE % (flow_id, dag_id, operator_id) - # ).json() - - # result = GetOperatorWithArtifactNodeResponse(**resp) - - # artifact_id = result.artifact_id - - # resp = self.get_response( - # self.LIST_ARTIFACT_RESULTS_TEMPLATE % (flow_id, artifact_id) - # ).json() - # results = resp["results"] - # # One of these should be correct for the DAG run and can get result content. - # for artifact_result in results: - # resp = self.get_response( - # self.GET_NODE_CHECK_RESULT_CONTENT_TEMPLATE - # % (flow_id, dag_id, operator_id, artifact_result["id"]) - # ) - # assert resp.ok - # resp_obj = GetNodeResultContentResponse(**resp.json()) - # # One of these should be successful (direct descendent of operator) - # assert not resp_obj.is_downsampled - # assert len(resp_obj.content) > 0 + def test_endpoint_node_check_get(self): + flow_id, _ = self.flows["flow_with_metrics_and_checks"] + flow = self.client.flow(flow_id) + workflow_resp = flow._get_workflow_resp() + dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + dag_result_id = workflow_resp.workflow_dag_results[0].id + + dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + flow_id, + dag_result_id, + ) + operator_ids = [ + id + for id in dag_result_resp.operators.keys() + if dag_result_resp.operators[id].spec.check + ] + operator_id = str(operator_ids[0]) + + resp = self.get_response( + self.GET_NODE_CHECK_TEMPLATE % (flow_id, dag_id, operator_id) + ).json() + result = GetOperatorWithArtifactNodeResponse(**resp) + assert str(result.id) == operator_id + assert result.dag_id == dag_id + assert len(result.inputs) == 1 + assert len(result.outputs) == 0 + + def test_endpoint_node_check_result_content_get(self): + flow_id, _ = self.flows["flow_with_metrics_and_checks"] + flow = self.client.flow(flow_id) + workflow_resp = flow._get_workflow_resp() + dag_id = workflow_resp.workflow_dag_results[0].workflow_dag_id + dag_result_id = workflow_resp.workflow_dag_results[0].id + + dag_result_resp = globals.__GLOBAL_API_CLIENT__.get_workflow_dag_result( + flow_id, + dag_result_id, + ) + operator_ids = [ + id + for id in dag_result_resp.operators.keys() + if dag_result_resp.operators[id].spec.check + ] + operator_id = str(operator_ids[0]) + + resp = self.get_response( + self.GET_NODE_CHECK_TEMPLATE % (flow_id, dag_id, operator_id) + ).json() + + result = GetOperatorWithArtifactNodeResponse(**resp) + + artifact_id = result.artifact_id + + resp = self.get_response( + self.LIST_ARTIFACT_RESULTS_TEMPLATE % (flow_id, artifact_id) + ).json() + results = resp["results"] + # One of these should be correct for the DAG run and can get result content. + for artifact_result in results: + resp = self.get_response( + self.GET_NODE_CHECK_RESULT_CONTENT_TEMPLATE + % (flow_id, dag_id, operator_id, artifact_result["id"]) + ) + assert resp.ok + resp_obj = GetNodeResultContentResponse(**resp.json()) + # One of these should be successful (direct descendent of operator) + assert not resp_obj.is_downsampled + assert len(resp_obj.content) > 0 def test_endpoint_node_check_results_get(self): flow_id = self.flows["flow_with_metrics_and_checks"][0] From 90a5db4e62aca147f70afe1785d0e9fe50d25ba4 Mon Sep 17 00:00:00 2001 From: Eunice Chan Date: Fri, 2 Jun 2023 14:29:47 -0700 Subject: [PATCH 5/6] Golang lint --- .../handler/v2/node_check_results_get.go | 4 ++-- .../handler/v2/node_metric_results_get.go | 4 ++-- src/golang/cmd/server/routes/routes.go | 2 +- .../operator_with_artifact_node_result.go | 20 +++++++++---------- .../lib/repos/sqlite/operator_result.go | 2 +- src/golang/lib/response/node.go | 14 ++++++------- 6 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/golang/cmd/server/handler/v2/node_check_results_get.go b/src/golang/cmd/server/handler/v2/node_check_results_get.go index 91bbd2becd..129789ced5 100644 --- a/src/golang/cmd/server/handler/v2/node_check_results_get.go +++ b/src/golang/cmd/server/handler/v2/node_check_results_get.go @@ -9,11 +9,11 @@ import ( "github.com/aqueducthq/aqueduct/lib/database" "github.com/aqueducthq/aqueduct/lib/models" "github.com/aqueducthq/aqueduct/lib/models/shared" + "github.com/aqueducthq/aqueduct/lib/models/views" "github.com/aqueducthq/aqueduct/lib/repos" "github.com/aqueducthq/aqueduct/lib/response" "github.com/aqueducthq/aqueduct/lib/storage" "github.com/dropbox/godropbox/errors" - "github.com/aqueducthq/aqueduct/lib/models/views" "github.com/google/uuid" ) @@ -121,7 +121,7 @@ func (h *NodeCheckResultsGetHandler) Perform(ctx context.Context, interfaceArgs var contentPtr *string = nil if !nodeResult.ArtifactResultExecState.IsNull && (nodeResult.ArtifactResultExecState.ExecutionState.Status == shared.FailedExecutionStatus || - nodeResult.ArtifactResultExecState.ExecutionState.Status == shared.SucceededExecutionStatus) { + nodeResult.ArtifactResultExecState.ExecutionState.Status == shared.SucceededExecutionStatus) { exists := storageObj.Exists(ctx, nodeResult.ContentPath) if exists { contentBytes, err := storageObj.Get(ctx, nodeResult.ContentPath) diff --git a/src/golang/cmd/server/handler/v2/node_metric_results_get.go b/src/golang/cmd/server/handler/v2/node_metric_results_get.go index 930b9957ec..4c6daaa106 100644 --- a/src/golang/cmd/server/handler/v2/node_metric_results_get.go +++ b/src/golang/cmd/server/handler/v2/node_metric_results_get.go @@ -9,11 +9,11 @@ import ( "github.com/aqueducthq/aqueduct/lib/database" "github.com/aqueducthq/aqueduct/lib/models" "github.com/aqueducthq/aqueduct/lib/models/shared" + "github.com/aqueducthq/aqueduct/lib/models/views" "github.com/aqueducthq/aqueduct/lib/repos" "github.com/aqueducthq/aqueduct/lib/response" "github.com/aqueducthq/aqueduct/lib/storage" "github.com/dropbox/godropbox/errors" - "github.com/aqueducthq/aqueduct/lib/models/views" "github.com/google/uuid" ) @@ -119,7 +119,7 @@ func (h *NodeMetricResultsGetHandler) Perform(ctx context.Context, interfaceArgs var contentPtr *string = nil if !nodeResult.ArtifactResultExecState.IsNull && (nodeResult.ArtifactResultExecState.ExecutionState.Status == shared.FailedExecutionStatus || - nodeResult.ArtifactResultExecState.ExecutionState.Status == shared.SucceededExecutionStatus) { + nodeResult.ArtifactResultExecState.ExecutionState.Status == shared.SucceededExecutionStatus) { exists := storageObj.Exists(ctx, nodeResult.ContentPath) if exists { contentBytes, err := storageObj.Get(ctx, nodeResult.ContentPath) diff --git a/src/golang/cmd/server/routes/routes.go b/src/golang/cmd/server/routes/routes.go index ba2d5fb2c0..8aaa13199d 100644 --- a/src/golang/cmd/server/routes/routes.go +++ b/src/golang/cmd/server/routes/routes.go @@ -17,7 +17,7 @@ const ( NodesRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/nodes" NodeArtifactRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/artifact/{nodeID}" NodeArtifactResultContentRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/artifact/{nodeID}/result/{nodeResultID}/content" - NodeArtifactResultsRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/artifact/{nodeID}/results" + NodeArtifactResultsRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/artifact/{nodeID}/results" NodeMetricRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/metric/{nodeID}" NodeMetricResultContentRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/metric/{nodeID}/result/{nodeResultID}/content" NodeMetricResultsRoute = "/api/v2/workflow/{workflowID}/dag/{dagID}/node/metric/{nodeID}/results" diff --git a/src/golang/lib/models/views/operator_with_artifact_node_result.go b/src/golang/lib/models/views/operator_with_artifact_node_result.go index c7383d64d0..4d2739e798 100644 --- a/src/golang/lib/models/views/operator_with_artifact_node_result.go +++ b/src/golang/lib/models/views/operator_with_artifact_node_result.go @@ -12,25 +12,25 @@ const ( OperatorWithArtifactResultNodeTable = "operator_with_artifact_node_result" // OperatorWithArtifactResultNode table column names - OperatorWithArtifactResultNodeID = "id" // operator result ID + OperatorWithArtifactResultNodeID = "id" // operator result ID OperatorWithArtifactResultNodeArtifactResultID = "artifact_result_id" - OperatorWithArtifactResultNodeOperatorID = "operator_id" - OperatorWithArtifactResultNodeArtifactID = "artifact_id" + OperatorWithArtifactResultNodeOperatorID = "operator_id" + OperatorWithArtifactResultNodeArtifactID = "artifact_id" OperatorWithArtifactResultNodeOperatorResultExecState = "operator_result_exec_state" - OperatorWithArtifactResultNodeMetadata = "metadata" - OperatorWithArtifactResultNodeContentPath = "content_path" + OperatorWithArtifactResultNodeMetadata = "metadata" + OperatorWithArtifactResultNodeContentPath = "content_path" OperatorWithArtifactResultNodeArtifactResultExecState = "artifact_result_exec_state" ) // An OperatorWithArtifactResultNode maps to the merged_node_result table. type OperatorWithArtifactResultNode struct { - ID uuid.UUID `db:"id" json:"id"` - OperatorID uuid.UUID `db:"operator_id" json:"operator_id"` + ID uuid.UUID `db:"id" json:"id"` + OperatorID uuid.UUID `db:"operator_id" json:"operator_id"` OperatorResultExecState shared.NullExecutionState `db:"operator_result_exec_state" json:"operator_result_exec_state"` - ArtifactID uuid.UUID `db:"artifact_id" json:"artifact_id"` + ArtifactID uuid.UUID `db:"artifact_id" json:"artifact_id"` ArtifactResultID uuid.UUID `db:"artifact_result_id" json:"artifact_result_id"` - Metadata shared.NullArtifactResultMetadata `db:"metadata" json:"metadata"` - ContentPath string `db:"content_path" json:"content_path"` + Metadata shared.NullArtifactResultMetadata `db:"metadata" json:"metadata"` + ContentPath string `db:"content_path" json:"content_path"` ArtifactResultExecState shared.NullExecutionState `db:"artifact_result_exec_state" json:"artifact_result_exec_state"` } diff --git a/src/golang/lib/repos/sqlite/operator_result.go b/src/golang/lib/repos/sqlite/operator_result.go index 63ce6e67fd..87efba0d47 100644 --- a/src/golang/lib/repos/sqlite/operator_result.go +++ b/src/golang/lib/repos/sqlite/operator_result.go @@ -186,7 +186,7 @@ func (*operatorResultReader) GetStatusByDAGResultAndArtifactBatch( func (*operatorResultReader) GetOperatorWithArtifactResultNodesByOperatorNameAndWorkflow( ctx context.Context, - operatorName string, + operatorName string, workflowID uuid.UUID, DB database.Database, ) ([]views.OperatorWithArtifactResultNode, error) { diff --git a/src/golang/lib/response/node.go b/src/golang/lib/response/node.go index 8e8a585d9b..35b6325a2a 100644 --- a/src/golang/lib/response/node.go +++ b/src/golang/lib/response/node.go @@ -45,13 +45,13 @@ func NewOperatorWithArtifactNodeFromDBObject(dbOperatorWithArtifactNode *views.O type OperatorWithArtifactResultNode struct { // Operator Result ID - ID uuid.UUID `json:"id"` + ID uuid.UUID `json:"id"` ArtifactResultID uuid.UUID `json:"artifact_result_id"` - OperatorID uuid.UUID `json:"operator_id"` - ArtifactID uuid.UUID `json:"artifact_id"` - OperatorResultExecState *shared.ExecutionState `json:"operator_result_exec_state"` - ArtifactResultExecState *shared.ExecutionState `json:"artifact_result_exec_state"` - SerializationType shared.ArtifactSerializationType `json:"serialization_type"` + OperatorID uuid.UUID `json:"operator_id"` + ArtifactID uuid.UUID `json:"artifact_id"` + OperatorResultExecState *shared.ExecutionState `json:"operator_result_exec_state"` + ArtifactResultExecState *shared.ExecutionState `json:"artifact_result_exec_state"` + SerializationType shared.ArtifactSerializationType `json:"serialization_type"` // If `ContentSerialized` is set, the content is small and we directly send // it as a part of response. It's consistent with the object stored in `ContentPath`. @@ -69,7 +69,7 @@ func NewOperatorWithArtifactResultNodeFromDBObject( ) *OperatorWithArtifactResultNode { result := &OperatorWithArtifactResultNode{ ID: dbOperatorWithArtifactResultNode.ID, - ArtifactResultID: dbOperatorWithArtifactResultNode.ArtifactResultID, + ArtifactResultID: dbOperatorWithArtifactResultNode.ArtifactResultID, OperatorID: dbOperatorWithArtifactResultNode.OperatorID, ArtifactID: dbOperatorWithArtifactResultNode.ArtifactID, SerializationType: dbOperatorWithArtifactResultNode.Metadata.SerializationType, From 92e1e4240cd8a2fa89a4b6a9e46c9873cbe81588 Mon Sep 17 00:00:00 2001 From: Eunice Chan Date: Fri, 2 Jun 2023 14:39:51 -0700 Subject: [PATCH 6/6] UI Lint --- src/ui/common/src/handlers/v2/NodeCheckResultsGet.ts | 7 +++---- src/ui/common/src/handlers/v2/NodeMetricResultsGet.ts | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ui/common/src/handlers/v2/NodeCheckResultsGet.ts b/src/ui/common/src/handlers/v2/NodeCheckResultsGet.ts index 254eca3eee..0b05bf4d2e 100644 --- a/src/ui/common/src/handlers/v2/NodeCheckResultsGet.ts +++ b/src/ui/common/src/handlers/v2/NodeCheckResultsGet.ts @@ -14,11 +14,10 @@ export type NodeCheckResultsGetRequest = APIKeyParameter & NodeIdParameter & WorkflowIdParameter; -export type NodeCheckResultsGetResponse = OperatorWithArtifactNodeResultResponse[]; +export type NodeCheckResultsGetResponse = + OperatorWithArtifactNodeResultResponse[]; -export const nodeCheckResultsGetQuery = ( - req: NodeCheckResultsGetRequest -) => ({ +export const nodeCheckResultsGetQuery = (req: NodeCheckResultsGetRequest) => ({ url: `workflow/${req.workflowId}/dag/${req.dagId}/node/check/${req.nodeId}/results`, headers: { 'api-key': req.apiKey }, }); diff --git a/src/ui/common/src/handlers/v2/NodeMetricResultsGet.ts b/src/ui/common/src/handlers/v2/NodeMetricResultsGet.ts index 16a7605ff9..2f96e4b989 100644 --- a/src/ui/common/src/handlers/v2/NodeMetricResultsGet.ts +++ b/src/ui/common/src/handlers/v2/NodeMetricResultsGet.ts @@ -14,7 +14,8 @@ export type NodeMetricResultsGetRequest = APIKeyParameter & NodeIdParameter & WorkflowIdParameter; -export type NodeMetricResultsGetResponse = OperatorWithArtifactNodeResultResponse[]; +export type NodeMetricResultsGetResponse = + OperatorWithArtifactNodeResultResponse[]; export const nodeMetricResultsGetQuery = ( req: NodeMetricResultsGetResponse