feat: add workflow macro UI support

common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/LogicalOp.scala

@@ -439,6 +439,9 @@ abstract class LogicalOp extends PortDescriptor with Serializable {
   @JsonProperty(PropertyNameConstants.OPERATOR_VERSION)
   var operatorVersion: String = getOperatorVersion
 
+  @JsonProperty(PropertyNameConstants.MACRO_ID_PARENT)
+  var macroIdParent: String = _
+
   def operatorIdentifier: OperatorIdentity = OperatorIdentity(operatorId)
 
   def getPhysicalOp(

common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/metadata/OperatorMetadataGenerator.scala

@@ -118,26 +118,24 @@ object OperatorMetadataGenerator {
 
   def generateOperatorJsonSchema(opDescClass: Class[_ <: LogicalOp]): JsonNode = {
     val jsonSchema = jsonSchemaGenerator.generateJsonSchema(opDescClass).asInstanceOf[ObjectNode]
-    // remove operatorID from json schema
-    jsonSchema.get("properties").asInstanceOf[ObjectNode].remove("operatorID")
-    // remove operatorId from json schema
-    jsonSchema.get("properties").asInstanceOf[ObjectNode].remove("operatorId")
-    // remove operatorType from json schema
-    jsonSchema.get("properties").asInstanceOf[ObjectNode].remove("operatorType")
-    // remove operatorVersion from json schema
-    jsonSchema.get("properties").asInstanceOf[ObjectNode].remove("operatorVersion")
-    // remove inputPorts/outputPorts from json schema
-    jsonSchema.get("properties").asInstanceOf[ObjectNode].remove("inputPorts")
-    jsonSchema.get("properties").asInstanceOf[ObjectNode].remove("outputPorts")
-    // remove operatorType from required list
-    val operatorTypeIndex =
-      jsonSchema
-        .get("required")
-        .asInstanceOf[ArrayNode]
-        .elements()
-        .asScala
-        .indexWhere(p => p.asText().equals("operatorType"))
-    jsonSchema.get("required").asInstanceOf[ArrayNode].remove(operatorTypeIndex)
+    val hiddenProperties = Seq(
+      "operatorID",
+      "operatorId",
+      "operatorType",
+      "operatorVersion",
+      "macroIdParent",
+      "inputPorts",
+      "outputPorts"
+    )
+    val properties = jsonSchema.get("properties").asInstanceOf[ObjectNode]
+    val required = jsonSchema.get("required").asInstanceOf[ArrayNode]
+    hiddenProperties.foreach { propertyName =>
+      properties.remove(propertyName)
+      val requiredIndex = required.elements().asScala.indexWhere(_.asText() == propertyName)
+      if (requiredIndex >= 0) {
+        required.remove(requiredIndex)
+      }
+    }
     // remove "title" for the operator - frontend uses userFriendlyName to show operator title
     jsonSchema.remove("title")
     jsonSchema

common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/metadata/PropertyNameConstants.scala

@@ -33,6 +33,7 @@ object PropertyNameConstants { // logical plan property names
   final val OPERATOR_LIST = "operators"
   final val OPERATOR_LINK_LIST = "links"
   final val OPERATOR_VERSION = "operatorVersion"
+  final val MACRO_ID_PARENT = "macroIdParent"
   // common operator property names
   final val ATTRIBUTE_NAMES = "attributes"
   final val ATTRIBUTE_NAME = "attribute"

frontend/src/app/common/type/workflow.ts

@@ -18,7 +18,13 @@
  */
 
 import { WorkflowMetadata } from "../../dashboard/type/workflow-metadata.interface";
-import { CommentBox, OperatorLink, OperatorPredicate, Point } from "../../workspace/types/workflow-common.interface";
+import {
+  CommentBox,
+  OperatorLink,
+  OperatorPredicate,
+  Point,
+  WorkflowMacro,
+} from "../../workspace/types/workflow-common.interface";
 
 export enum ExecutionMode {
   PIPELINED = "PIPELINED",
@@ -49,6 +55,7 @@ export interface WorkflowContent
     links: OperatorLink[];
     commentBoxes: CommentBox[];
     settings: WorkflowSettings;
+    macros?: WorkflowMacro[];
   }> {}
 
 export type Workflow = { content: WorkflowContent } & WorkflowMetadata;

frontend/src/app/workspace/component/left-panel/operator-menu/operator-menu.component.ts

@@ -36,6 +36,8 @@ import { FormsModule } from "@angular/forms";
 import { NgFor, NgTemplateOutlet } from "@angular/common";
 import { OperatorLabelComponent } from "./operator-label/operator-label.component";
 import { NzCollapseComponent, NzCollapsePanelComponent } from "ng-zorro-antd/collapse";
+import { WORKFLOW_MACRO_OPERATOR_TYPE } from "../../../service/operator-metadata/operator-metadata.service";
+import { JointGraphWrapper } from "../../../service/workflow-graph/model/joint-graph-wrapper";
 
 @UntilDestroy()
 @Component({
@@ -138,13 +140,23 @@ export class OperatorMenuComponent {
     // add the operator to the graph on select (position relative to the current viewpoint)
     const origin = this.workflowActionService.getJointGraphWrapper().getMainJointPaper()?.translate();
     const point = { x: 400 - (origin?.tx ?? 0), y: 200 - (origin?.ty ?? 0) };
+    if (selectSchema.operatorType === WORKFLOW_MACRO_OPERATOR_TYPE) {
+      const macroID = this.workflowActionService.createMacroAt(point);
+      this.workflowActionService.getJointGraphWrapper().highlightOperators(JointGraphWrapper.getMacroNodeID(macroID));
+      this.clearSearch();
+      return;
+    }
     this.workflowActionService.addOperator(
       this.workflowUtilService.getNewOperatorPredicate(selectSchema.operatorType),
       point
     );
 
     // asynchronously immediately clear the search input and suggestions
     // because ng-zorro shows the selected value if it's synchronously
+    this.clearSearch();
+  }
+
+  private clearSearch(): void {
     setTimeout(() => {
       this.searchInputValue = "";
       this.autocompleteOptions = [];

frontend/src/app/workspace/component/property-editor/macro-property-edit-frame/macro-property-edit-frame.component.html

@@ -0,0 +1,54 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<h3 class="texera-workspace-property-editor-title">Workflow Macro</h3>
+
+<label class="macro-field-label">Workflow</label>
+<nz-select
+  nzShowSearch
+  nzPlaceHolder="Select workflow"
+  [nzLoading]="loading"
+  [nzDisabled]="!canModify || loading"
+  [(ngModel)]="selectedWorkflowId"
+  (ngModelChange)="onWorkflowChange($event)">
+  <nz-option
+    *ngFor="let entry of workflows"
+    [nzValue]="entry.workflow.wid"
+    [nzLabel]="entry.workflow.name">
+  </nz-option>
+</nz-select>
+
+<div
+  *ngIf="macro?.workflowName"
+  class="macro-selected-workflow">
+  #{{ macro?.workflowId }} {{ macro?.workflowName }}
+</div>
+
+<div class="macro-actions">
+  <button
+    nz-button
+    nzType="default"
+    [disabled]="!macro || !canModify || loading"
+    (click)="toggleCollapsed()">
+    <span
+      nz-icon
+      [nzType]="macro?.collapsed ? 'fullscreen' : 'fullscreen-exit'"></span>
+    {{ macro?.collapsed ? "Expand internals" : "Collapse internals" }}
+  </button>
+</div>

frontend/src/app/workspace/component/property-editor/macro-property-edit-frame/macro-property-edit-frame.component.scss

@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+.macro-field-label {
+  display: block;
+  margin-bottom: 6px;
+  font-weight: 600;
+}
+
+nz-select {
+  width: 100%;
+}
+
+.macro-selected-workflow {
+  margin-top: 10px;
+  color: #666;
+  font-size: 12px;
+}
+
+.macro-actions {
+  margin-top: 14px;
+}
+
+.macro-actions button {
+  width: 100%;
+}

frontend/src/app/workspace/component/property-editor/macro-property-edit-frame/macro-property-edit-frame.component.ts

@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { Component, Input, OnChanges, OnDestroy, OnInit } from "@angular/core";
+import { FormsModule } from "@angular/forms";
+import { NgFor, NgIf } from "@angular/common";
+import { NzOptionComponent, NzSelectComponent } from "ng-zorro-antd/select";
+import { NzButtonComponent } from "ng-zorro-antd/button";
+import { NzIconDirective } from "ng-zorro-antd/icon";
+import { Subject } from "rxjs";
+import { finalize, take, takeUntil } from "rxjs/operators";
+import { WorkflowPersistService } from "../../../../common/service/workflow-persist/workflow-persist.service";
+import { DashboardWorkflow } from "../../../../dashboard/type/dashboard-workflow.interface";
+import { NotificationService } from "../../../../common/service/notification/notification.service";
+import { WorkflowActionService } from "../../../service/workflow-graph/model/workflow-action.service";
+import { JointGraphWrapper } from "../../../service/workflow-graph/model/joint-graph-wrapper";
+import { WorkflowMacro } from "../../../types/workflow-common.interface";
+
+@Component({
+  selector: "texera-macro-property-edit-frame",
+  templateUrl: "./macro-property-edit-frame.component.html",
+  styleUrls: ["./macro-property-edit-frame.component.scss"],
+  imports: [FormsModule, NgFor, NgIf, NzSelectComponent, NzOptionComponent, NzButtonComponent, NzIconDirective],
+})
+export class MacroPropertyEditFrameComponent implements OnInit, OnChanges, OnDestroy {
+  @Input() macroNodeId?: string;
+
+  public workflows: DashboardWorkflow[] = [];
+  public selectedWorkflowId?: number;
+  public loading = false;
+  public canModify = true;
+  private macroID?: string;
+  private destroy$ = new Subject<void>();
+
+  constructor(
+    private workflowPersistService: WorkflowPersistService,
+    private workflowActionService: WorkflowActionService,
+    private notificationService: NotificationService
+  ) {}
+
+  ngOnInit(): void {
+    this.canModify = this.workflowActionService.checkWorkflowModificationEnabled();
+    this.workflowActionService
+      .getWorkflowModificationEnabledStream()
+      .pipe(takeUntil(this.destroy$))
+      .subscribe(canModify => (this.canModify = canModify));
+
+    this.workflowPersistService
+      .retrieveWorkflowsBySessionUser()
+      .pipe(take(1))
+      .subscribe(workflows => (this.workflows = workflows));
+  }
+
+  ngOnChanges(): void {
+    if (!this.macroNodeId) return;
+    this.macroID = JointGraphWrapper.getMacroIDFromNodeID(this.macroNodeId);
+    this.selectedWorkflowId = this.macro?.workflowId;
+  }
+
+  public get macro(): WorkflowMacro | undefined {
+    return this.macroID ? this.workflowActionService.getWorkflowMacro(this.macroID) : undefined;
+  }
+
+  public onWorkflowChange(workflowId: number | undefined): void {
+    if (!this.canModify || !this.macroID || workflowId === undefined) return;
+    this.loading = true;
+    this.workflowPersistService
+      .retrieveWorkflow(workflowId)
+      .pipe(
+        take(1),
+        finalize(() => (this.loading = false))
+      )
+      .subscribe({
+        next: workflow => {
+          this.workflowActionService.replaceMacroWorkflow(this.macroID!, workflow.content, workflow.wid, workflow.name);
+          this.notificationService.info(`Imported workflow "${workflow.name}" into macro.`);
+        },
+        error: () => this.notificationService.error("Failed to import workflow into macro."),
+      });
+  }
+
+  public toggleCollapsed(): void {
+    if (!this.canModify || !this.macroID || !this.macro) return;
+    this.workflowActionService.setMacroCollapsed(this.macroID, !(this.macro.collapsed ?? false));
+  }
+
+  ngOnDestroy(): void {
+    this.destroy$.next();
+    this.destroy$.complete();
+  }
+}

frontend/src/app/workspace/component/property-editor/property-editor.component.ts

@@ -45,6 +45,8 @@ import { CdkDrag, CdkDragHandle } from "@angular/cdk/drag-drop";
 import { NzSpaceCompactItemDirective } from "ng-zorro-antd/space";
 import { NzButtonComponent } from "ng-zorro-antd/button";
 import { FormlyRepeatDndComponent } from "../../../common/formly/repeat-dnd/repeat-dnd.component";
+import { JointGraphWrapper } from "../../service/workflow-graph/model/joint-graph-wrapper";
+import { MacroPropertyEditFrameComponent } from "./macro-property-edit-frame/macro-property-edit-frame.component";
 
 /**
  * PropertyEditorComponent is the panel that allows user to edit operator properties.
@@ -74,6 +76,7 @@ import { FormlyRepeatDndComponent } from "../../../common/formly/repeat-dnd/repe
     NgComponentOutlet,
     NzResizeHandlesComponent,
     FormlyRepeatDndComponent,
+    MacroPropertyEditFrameComponent,
   ],
 })
 export class PropertyEditorComponent implements OnInit, OnDestroy {
@@ -165,7 +168,15 @@ export class PropertyEditorComponent implements OnInit, OnDestroy {
         this.workflowActionService.getJointGraphWrapper().getCurrentHighlightedCommentBoxIDs();
         const highlightedPorts = this.workflowActionService.getJointGraphWrapper().getCurrentHighlightedPortIDs();
 
-        if (highlightedOperators.length === 1 && highlightLinks.length === 0 && highlightedPorts.length === 0) {
+        if (
+          highlightedOperators.length === 1 &&
+          highlightLinks.length === 0 &&
+          highlightedPorts.length === 0 &&
+          JointGraphWrapper.isMacroNodeID(highlightedOperators[0])
+        ) {
+          this.currentComponent = MacroPropertyEditFrameComponent;
+          this.componentInputs = { macroNodeId: highlightedOperators[0] };
+        } else if (highlightedOperators.length === 1 && highlightLinks.length === 0 && highlightedPorts.length === 0) {
           this.currentComponent = OperatorPropertyEditFrameComponent;
           this.componentInputs = { currentOperatorId: highlightedOperators[0] };
         } else if (highlightedPorts.length === 1 && highlightLinks.length === 0) {