diff --git a/airflow-core/src/airflow/api_fastapi/common/parameters.py b/airflow-core/src/airflow/api_fastapi/common/parameters.py index e4c7b1bfd5d53..a6eff80544a06 100644 --- a/airflow-core/src/airflow/api_fastapi/common/parameters.py +++ b/airflow-core/src/airflow/api_fastapi/common/parameters.py @@ -1044,6 +1044,43 @@ def depends(cls, teams: list[str] = Query(default_factory=list)) -> _TeamsFilter return cls().set_value(teams) +class _RelativeFilelocPrefixFilter(BaseParam[str | None]): + """ + Filter Dags by the folder they live in, derived from ``relative_fileloc``. + + The value is treated as a directory path relative to the bundle root (e.g. + ``team_a/etl``). It matches every Dag whose file lives directly in that folder + or in any of its subfolders, using an escaped ``LIKE 'team_a/etl/%'`` so a + folder name is never a substring/prefix of another (``team_a`` won't match + ``team_alpha``). Dags at the bundle root (no ``/`` in ``relative_fileloc``) + are not matched by any folder value and appear only when no folder is selected. + """ + + def to_orm(self, select: Select) -> Select: + if self.value is None and self.skip_none: + return select + + if not self.value: + return select + + directory = self.value.rstrip("/") + escaped = _escape_like_pattern(directory) + return select.where(DagModel.relative_fileloc.like(f"{escaped}/%", escape=_LIKE_ESCAPE_CHAR)) + + @classmethod + def depends( + cls, + relative_fileloc_prefix: str | None = Query( + default=None, + description=( + "Filter Dags by the folder (directory of ``relative_fileloc``) they live in. " + "Matches the given folder and all of its subfolders." + ), + ), + ) -> _RelativeFilelocPrefixFilter: + return cls().set_value(relative_fileloc_prefix) + + def _safe_parse_datetime(date_to_check: str) -> datetime: """ Parse datetime and raise error for invalid dates. @@ -1293,6 +1330,9 @@ def depends_float( QueryTagsFilter = Annotated[_TagsFilter, Depends(_TagsFilter.depends)] QueryOwnersFilter = Annotated[_OwnersFilter, Depends(_OwnersFilter.depends)] QueryTeamsFilter = Annotated[_TeamsFilter, Depends(_TeamsFilter.depends)] +QueryRelativeFilelocPrefixFilter = Annotated[ + _RelativeFilelocPrefixFilter, Depends(_RelativeFilelocPrefixFilter.depends) +] class _HasAssetScheduleFilter(BaseParam[bool]): diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dags.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dags.py index e49e89087daeb..b9e8c614a7b79 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dags.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/dags.py @@ -54,3 +54,17 @@ class DAGsRunStateCountsCollectionResponse(BaseModel): dags: list[DAGRunStateCountsResponse] state_count_limit: int + + +class DagFolderResponse(BaseModel): + """A distinct Dag folder (directory of ``relative_fileloc``) within a bundle.""" + + bundle_name: str + folder: str + + +class DagFolderCollectionResponse(BaseModel): + """Collection of distinct Dag folders, each scoped to the bundle it belongs to.""" + + folders: list[DagFolderResponse] + total_entries: int diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml index e34789c3524d0..daea9e3132228 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml @@ -509,6 +509,18 @@ paths: - type: string - type: 'null' title: Bundle Version + - name: relative_fileloc_prefix + in: query + required: false + schema: + anyOf: + - type: string + - type: 'null' + description: Filter Dags by the folder (directory of ``relative_fileloc``) + they live in. Matches the given folder and all of its subfolders. + title: Relative Fileloc Prefix + description: Filter Dags by the folder (directory of ``relative_fileloc``) + they live in. Matches the given folder and all of its subfolders. - name: order_by in: query required: false @@ -574,6 +586,43 @@ paths: application/json: schema: $ref: '#/components/schemas/HTTPValidationError' + /ui/dags/folders: + get: + tags: + - DAG + summary: Get Dag Folders + description: 'Get the distinct folders the readable Dags live in, scoped to + their bundle. + + + A folder is the directory part of a Dag''s ``relative_fileloc`` (relative + to its + + bundle root). Because ``relative_fileloc`` is relative to each bundle, the + same + + path can exist in several bundles, so every folder is paired with its bundle + + name to keep them apart. Dags located directly at the bundle root have no + folder + + and are not represented here. The result powers the folder navigation tree + in + + the UI, which reconstructs the hierarchy by splitting each path on ``/`` and + + groups it under its bundle when more than one bundle is present.' + operationId: get_dag_folders + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/DagFolderCollectionResponse' + security: + - OAuth2PasswordBearer: [] + - HTTPBearer: [] /ui/dags/{dag_id}/latest_run: get: tags: @@ -2706,6 +2755,38 @@ components: - state_count_limit title: DAGsRunStateCountsCollectionResponse description: Collection of per-Dag DagRun-state counts for the Dag list page. + DagFolderCollectionResponse: + properties: + folders: + items: + $ref: '#/components/schemas/DagFolderResponse' + type: array + title: Folders + total_entries: + type: integer + title: Total Entries + type: object + required: + - folders + - total_entries + title: DagFolderCollectionResponse + description: Collection of distinct Dag folders, each scoped to the bundle it + belongs to. + DagFolderResponse: + properties: + bundle_name: + type: string + title: Bundle Name + folder: + type: string + title: Folder + type: object + required: + - bundle_name + - folder + title: DagFolderResponse + description: A distinct Dag folder (directory of ``relative_fileloc``) within + a bundle. DagRunState: type: string enum: diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py index 38421d387b10a..dbe48cfc4c79e 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py @@ -17,6 +17,7 @@ from __future__ import annotations +from pathlib import PurePosixPath from typing import Annotated from fastapi import Depends, HTTPException, Query, status @@ -50,6 +51,7 @@ QueryOwnersFilter, QueryPausedFilter, QueryPendingActionsFilter, + QueryRelativeFilelocPrefixFilter, QueryTagsFilter, QueryTeamsFilter, SortParam, @@ -59,6 +61,8 @@ from airflow.api_fastapi.core_api.datamodels.dags import DAG_ALIAS_MAPPING, DAGResponse from airflow.api_fastapi.core_api.datamodels.ui.dag_runs import DAGRunLightResponse from airflow.api_fastapi.core_api.datamodels.ui.dags import ( + DagFolderCollectionResponse, + DagFolderResponse, DAGRunStateCountsResponse, DAGsRunStateCountsCollectionResponse, DAGWithLatestDagRunsCollectionResponse, @@ -113,6 +117,7 @@ def get_dags( dag_run_state: QueryAnyDagRunStateFilter, bundle_name: QueryBundleNameFilter, bundle_version: QueryBundleVersionFilter, + relative_fileloc_prefix: QueryRelativeFilelocPrefixFilter, order_by: Annotated[ SortParam, Depends( @@ -169,6 +174,7 @@ def get_dags( readable_dags_filter, bundle_name, bundle_version, + relative_fileloc_prefix, ], order_by=order_by, offset=offset, @@ -275,6 +281,47 @@ def get_dags( ) +@dags_router.get( + "/folders", + dependencies=[Depends(requires_access_dag(method="GET"))], + operation_id="get_dag_folders", +) +def get_dag_folders( + readable_dags_filter: ReadableDagsFilterDep, + session: SessionDep, +) -> DagFolderCollectionResponse: + """ + Get the distinct folders the readable Dags live in, scoped to their bundle. + + A folder is the directory part of a Dag's ``relative_fileloc`` (relative to its + bundle root). Because ``relative_fileloc`` is relative to each bundle, the same + path can exist in several bundles, so every folder is paired with its bundle + name to keep them apart. Dags located directly at the bundle root have no folder + and are not represented here. The result powers the folder navigation tree in + the UI, which reconstructs the hierarchy by splitting each path on ``/`` and + groups it under its bundle when more than one bundle is present. + """ + query = readable_dags_filter.to_orm( + select(DagModel.bundle_name, DagModel.relative_fileloc) + .where(DagModel.relative_fileloc.is_not(None)) + .distinct() + ) + folders: set[tuple[str, str]] = set() + for bundle_name, relative_fileloc in session.execute(query): + parent = PurePosixPath(relative_fileloc).parent + if str(parent) != ".": + folders.add((bundle_name, str(parent))) + + sorted_folders = sorted(folders) + return DagFolderCollectionResponse( + folders=[ + DagFolderResponse(bundle_name=bundle_name, folder=folder) + for bundle_name, folder in sorted_folders + ], + total_entries=len(sorted_folders), + ) + + @dags_router.get( "/{dag_id}/latest_run", responses=create_openapi_http_exception_doc([status.HTTP_404_NOT_FOUND]), diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts index 1cf5105afedc8..852c215b4726f 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/common.ts @@ -339,7 +339,7 @@ export const UseDagServiceGetDagTagsKeyFn = ({ limit, offset, orderBy, tagNamePa export type DagServiceGetDagsUiDefaultResponse = Awaited>; export type DagServiceGetDagsUiQueryResult = UseQueryResult; export const useDagServiceGetDagsUiKey = "DagServiceGetDagsUi"; -export const UseDagServiceGetDagsUiKeyFn = ({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }: { +export const UseDagServiceGetDagsUiKeyFn = ({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }: { assetDependency?: string; bundleName?: string; bundleVersion?: string; @@ -361,10 +361,15 @@ export const UseDagServiceGetDagsUiKeyFn = ({ assetDependency, bundleName, bundl orderBy?: string[]; owners?: string[]; paused?: boolean; + relativeFilelocPrefix?: string; tags?: string[]; tagsMatchMode?: "any" | "all"; teams?: string[]; -} = {}, queryKey?: Array) => [useDagServiceGetDagsUiKey, ...(queryKey ?? [{ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }])]; +} = {}, queryKey?: Array) => [useDagServiceGetDagsUiKey, ...(queryKey ?? [{ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }])]; +export type DagServiceGetDagFoldersDefaultResponse = Awaited>; +export type DagServiceGetDagFoldersQueryResult = UseQueryResult; +export const useDagServiceGetDagFoldersKey = "DagServiceGetDagFolders"; +export const UseDagServiceGetDagFoldersKeyFn = (queryKey?: Array) => [useDagServiceGetDagFoldersKey, ...(queryKey ?? [])]; export type DagServiceGetLatestRunInfoDefaultResponse = Awaited>; export type DagServiceGetLatestRunInfoQueryResult = UseQueryResult; export const useDagServiceGetLatestRunInfoKey = "DagServiceGetLatestRunInfo"; diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts index 8fbebba6f50a2..0f5356816d331 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/ensureQueryData.ts @@ -692,6 +692,7 @@ export const ensureUseDagServiceGetDagTagsData = (queryClient: QueryClient, { li * @param data.dagRunState Filter Dags that have any DagRun in the given state. Only ``queued`` and ``running`` are supported. * @param data.bundleName * @param data.bundleVersion +* @param data.relativeFilelocPrefix Filter Dags by the folder (directory of ``relative_fileloc``) they live in. Matches the given folder and all of its subfolders. * @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `dag_id, dag_display_name, next_dagrun, state, start_date, last_run_state, last_run_start_date, last_run_run_after` * @param data.isFavorite * @param data.hasAssetSchedule Filter Dags with asset-based scheduling @@ -700,7 +701,7 @@ export const ensureUseDagServiceGetDagTagsData = (queryClient: QueryClient, { li * @returns DAGWithLatestDagRunsCollectionResponse Successful Response * @throws ApiError */ -export const ensureUseDagServiceGetDagsUiData = (queryClient: QueryClient, { assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }: { +export const ensureUseDagServiceGetDagsUiData = (queryClient: QueryClient, { assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }: { assetDependency?: string; bundleName?: string; bundleVersion?: string; @@ -722,10 +723,26 @@ export const ensureUseDagServiceGetDagsUiData = (queryClient: QueryClient, { ass orderBy?: string[]; owners?: string[]; paused?: boolean; + relativeFilelocPrefix?: string; tags?: string[]; tagsMatchMode?: "any" | "all"; teams?: string[]; -} = {}) => queryClient.ensureQueryData({ queryKey: Common.UseDagServiceGetDagsUiKeyFn({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }), queryFn: () => DagService.getDagsUi({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }) }); +} = {}) => queryClient.ensureQueryData({ queryKey: Common.UseDagServiceGetDagsUiKeyFn({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }), queryFn: () => DagService.getDagsUi({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }) }); +/** +* Get Dag Folders +* Get the distinct folders the readable Dags live in, scoped to their bundle. +* +* A folder is the directory part of a Dag's ``relative_fileloc`` (relative to its +* bundle root). Because ``relative_fileloc`` is relative to each bundle, the same +* path can exist in several bundles, so every folder is paired with its bundle +* name to keep them apart. Dags located directly at the bundle root have no folder +* and are not represented here. The result powers the folder navigation tree in +* the UI, which reconstructs the hierarchy by splitting each path on ``/`` and +* groups it under its bundle when more than one bundle is present. +* @returns DagFolderCollectionResponse Successful Response +* @throws ApiError +*/ +export const ensureUseDagServiceGetDagFoldersData = (queryClient: QueryClient) => queryClient.ensureQueryData({ queryKey: Common.UseDagServiceGetDagFoldersKeyFn(), queryFn: () => DagService.getDagFolders() }); /** * Get Latest Run Info * Get latest run. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts index e6937375f25b9..f32a2f16f03ca 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/prefetch.ts @@ -692,6 +692,7 @@ export const prefetchUseDagServiceGetDagTags = (queryClient: QueryClient, { limi * @param data.dagRunState Filter Dags that have any DagRun in the given state. Only ``queued`` and ``running`` are supported. * @param data.bundleName * @param data.bundleVersion +* @param data.relativeFilelocPrefix Filter Dags by the folder (directory of ``relative_fileloc``) they live in. Matches the given folder and all of its subfolders. * @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `dag_id, dag_display_name, next_dagrun, state, start_date, last_run_state, last_run_start_date, last_run_run_after` * @param data.isFavorite * @param data.hasAssetSchedule Filter Dags with asset-based scheduling @@ -700,7 +701,7 @@ export const prefetchUseDagServiceGetDagTags = (queryClient: QueryClient, { limi * @returns DAGWithLatestDagRunsCollectionResponse Successful Response * @throws ApiError */ -export const prefetchUseDagServiceGetDagsUi = (queryClient: QueryClient, { assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }: { +export const prefetchUseDagServiceGetDagsUi = (queryClient: QueryClient, { assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }: { assetDependency?: string; bundleName?: string; bundleVersion?: string; @@ -722,10 +723,26 @@ export const prefetchUseDagServiceGetDagsUi = (queryClient: QueryClient, { asset orderBy?: string[]; owners?: string[]; paused?: boolean; + relativeFilelocPrefix?: string; tags?: string[]; tagsMatchMode?: "any" | "all"; teams?: string[]; -} = {}) => queryClient.prefetchQuery({ queryKey: Common.UseDagServiceGetDagsUiKeyFn({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }), queryFn: () => DagService.getDagsUi({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }) }); +} = {}) => queryClient.prefetchQuery({ queryKey: Common.UseDagServiceGetDagsUiKeyFn({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }), queryFn: () => DagService.getDagsUi({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }) }); +/** +* Get Dag Folders +* Get the distinct folders the readable Dags live in, scoped to their bundle. +* +* A folder is the directory part of a Dag's ``relative_fileloc`` (relative to its +* bundle root). Because ``relative_fileloc`` is relative to each bundle, the same +* path can exist in several bundles, so every folder is paired with its bundle +* name to keep them apart. Dags located directly at the bundle root have no folder +* and are not represented here. The result powers the folder navigation tree in +* the UI, which reconstructs the hierarchy by splitting each path on ``/`` and +* groups it under its bundle when more than one bundle is present. +* @returns DagFolderCollectionResponse Successful Response +* @throws ApiError +*/ +export const prefetchUseDagServiceGetDagFolders = (queryClient: QueryClient) => queryClient.prefetchQuery({ queryKey: Common.UseDagServiceGetDagFoldersKeyFn(), queryFn: () => DagService.getDagFolders() }); /** * Get Latest Run Info * Get latest run. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts index c586eabf179eb..7ea6fbf920777 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/queries.ts @@ -692,6 +692,7 @@ export const useDagServiceGetDagTags = = unknown[]>({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }: { +export const useDagServiceGetDagsUi = = unknown[]>({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }: { assetDependency?: string; bundleName?: string; bundleVersion?: string; @@ -722,10 +723,26 @@ export const useDagServiceGetDagsUi = , "queryKey" | "queryFn">) => useQuery({ queryKey: Common.UseDagServiceGetDagsUiKeyFn({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }, queryKey), queryFn: () => DagService.getDagsUi({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }) as TData, ...options }); +} = {}, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useQuery({ queryKey: Common.UseDagServiceGetDagsUiKeyFn({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }, queryKey), queryFn: () => DagService.getDagsUi({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }) as TData, ...options }); +/** +* Get Dag Folders +* Get the distinct folders the readable Dags live in, scoped to their bundle. +* +* A folder is the directory part of a Dag's ``relative_fileloc`` (relative to its +* bundle root). Because ``relative_fileloc`` is relative to each bundle, the same +* path can exist in several bundles, so every folder is paired with its bundle +* name to keep them apart. Dags located directly at the bundle root have no folder +* and are not represented here. The result powers the folder navigation tree in +* the UI, which reconstructs the hierarchy by splitting each path on ``/`` and +* groups it under its bundle when more than one bundle is present. +* @returns DagFolderCollectionResponse Successful Response +* @throws ApiError +*/ +export const useDagServiceGetDagFolders = = unknown[]>(queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useQuery({ queryKey: Common.UseDagServiceGetDagFoldersKeyFn(queryKey), queryFn: () => DagService.getDagFolders() as TData, ...options }); /** * Get Latest Run Info * Get latest run. diff --git a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts index 3bc869e2b7d96..9a3c3597888cc 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/queries/suspense.ts @@ -692,6 +692,7 @@ export const useDagServiceGetDagTagsSuspense = = unknown[]>({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }: { +export const useDagServiceGetDagsUiSuspense = = unknown[]>({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }: { assetDependency?: string; bundleName?: string; bundleVersion?: string; @@ -722,10 +723,26 @@ export const useDagServiceGetDagsUiSuspense = , "queryKey" | "queryFn">) => useSuspenseQuery({ queryKey: Common.UseDagServiceGetDagsUiKeyFn({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }, queryKey), queryFn: () => DagService.getDagsUi({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, tags, tagsMatchMode, teams }) as TData, ...options }); +} = {}, queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useSuspenseQuery({ queryKey: Common.UseDagServiceGetDagsUiKeyFn({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }, queryKey), queryFn: () => DagService.getDagsUi({ assetDependency, bundleName, bundleVersion, dagDisplayNamePattern, dagDisplayNamePrefixPattern, dagIdPattern, dagIdPrefixPattern, dagIds, dagRunsLimit, dagRunState, excludeStale, hasAssetSchedule, hasImportErrors, hasPendingActions, isFavorite, lastDagRunState, limit, offset, orderBy, owners, paused, relativeFilelocPrefix, tags, tagsMatchMode, teams }) as TData, ...options }); +/** +* Get Dag Folders +* Get the distinct folders the readable Dags live in, scoped to their bundle. +* +* A folder is the directory part of a Dag's ``relative_fileloc`` (relative to its +* bundle root). Because ``relative_fileloc`` is relative to each bundle, the same +* path can exist in several bundles, so every folder is paired with its bundle +* name to keep them apart. Dags located directly at the bundle root have no folder +* and are not represented here. The result powers the folder navigation tree in +* the UI, which reconstructs the hierarchy by splitting each path on ``/`` and +* groups it under its bundle when more than one bundle is present. +* @returns DagFolderCollectionResponse Successful Response +* @throws ApiError +*/ +export const useDagServiceGetDagFoldersSuspense = = unknown[]>(queryKey?: TQueryKey, options?: Omit, "queryKey" | "queryFn">) => useSuspenseQuery({ queryKey: Common.UseDagServiceGetDagFoldersKeyFn(queryKey), queryFn: () => DagService.getDagFolders() as TData, ...options }); /** * Get Latest Run Info * Get latest run. diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts index fc4f1711c40e2..da72bc8e9611b 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -9481,6 +9481,43 @@ export const $DAGsRunStateCountsCollectionResponse = { description: 'Collection of per-Dag DagRun-state counts for the Dag list page.' } as const; +export const $DagFolderCollectionResponse = { + properties: { + folders: { + items: { + '$ref': '#/components/schemas/DagFolderResponse' + }, + type: 'array', + title: 'Folders' + }, + total_entries: { + type: 'integer', + title: 'Total Entries' + } + }, + type: 'object', + required: ['folders', 'total_entries'], + title: 'DagFolderCollectionResponse', + description: 'Collection of distinct Dag folders, each scoped to the bundle it belongs to.' +} as const; + +export const $DagFolderResponse = { + properties: { + bundle_name: { + type: 'string', + title: 'Bundle Name' + }, + folder: { + type: 'string', + title: 'Folder' + } + }, + type: 'object', + required: ['bundle_name', 'folder'], + title: 'DagFolderResponse', + description: 'A distinct Dag folder (directory of ``relative_fileloc``) within a bundle.' +} as const; + export const $DagRunStatsResponse = { properties: { duration: { diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts index 3ae4565069cc1..093117eaf8add 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/services.gen.ts @@ -3,7 +3,7 @@ import type { CancelablePromise } from './core/CancelablePromise'; import { OpenAPI } from './core/OpenAPI'; import { request as __request } from './core/request'; -import type { GetAssetsData, GetAssetsResponse, GetAssetAliasesData, GetAssetAliasesResponse, GetAssetAliasData, GetAssetAliasResponse, GetAssetEventsData, GetAssetEventsResponse, CreateAssetEventData, CreateAssetEventResponse, MaterializeAssetData, MaterializeAssetResponse, GetAssetQueuedEventsData, GetAssetQueuedEventsResponse, DeleteAssetQueuedEventsData, DeleteAssetQueuedEventsResponse, GetAssetData, GetAssetResponse, GetDagAssetQueuedEventsData, GetDagAssetQueuedEventsResponse, DeleteDagAssetQueuedEventsData, DeleteDagAssetQueuedEventsResponse, GetDagAssetQueuedEventData, GetDagAssetQueuedEventResponse, DeleteDagAssetQueuedEventData, DeleteDagAssetQueuedEventResponse, NextRunAssetsData, NextRunAssetsResponse2, ListBackfillsData, ListBackfillsResponse, CreateBackfillData, CreateBackfillResponse, GetBackfillData, GetBackfillResponse, ListBackfillDagRunsData, ListBackfillDagRunsResponse, PauseBackfillData, PauseBackfillResponse, UnpauseBackfillData, UnpauseBackfillResponse, CancelBackfillData, CancelBackfillResponse, CreateBackfillDryRunData, CreateBackfillDryRunResponse, ListBackfillsUiData, ListBackfillsUiResponse, DeleteConnectionData, DeleteConnectionResponse, GetConnectionData, GetConnectionResponse, PatchConnectionData, PatchConnectionResponse, GetConnectionTestData, GetConnectionTestResponse, EnqueueConnectionTestData, EnqueueConnectionTestResponse, GetConnectionsData, GetConnectionsResponse, PostConnectionData, PostConnectionResponse, BulkConnectionsData, BulkConnectionsResponse, TestConnectionData, TestConnectionResponse, CreateDefaultConnectionsResponse, HookMetaDataResponse, GetDagRunData, GetDagRunResponse, DeleteDagRunData, DeleteDagRunResponse, PatchDagRunData, PatchDagRunResponse, BulkDagRunsData, BulkDagRunsResponse, GetDagRunsData, GetDagRunsResponse, TriggerDagRunData, TriggerDagRunResponse, GetUpstreamAssetEventsData, GetUpstreamAssetEventsResponse, ClearDagRunData, ClearDagRunResponse, WaitDagRunUntilFinishedData, WaitDagRunUntilFinishedResponse, GetListDagRunsBatchData, GetListDagRunsBatchResponse, ClearDagRunsData, ClearDagRunsResponse, ClearDagRunPartitionsData, ClearDagRunPartitionsResponse, GetDagRunStatsData, GetDagRunStatsResponse, GetDagSourceData, GetDagSourceResponse, GetDagStatsData, GetDagStatsResponse, GetConfigData, GetConfigResponse, GetConfigValueData, GetConfigValueResponse, GetConfigsResponse, ListDagWarningsData, ListDagWarningsResponse, GetDagsData, GetDagsResponse, PatchDagsData, PatchDagsResponse, GetDagData, GetDagResponse, PatchDagData, PatchDagResponse, DeleteDagData, DeleteDagResponse, GetDagDetailsData, GetDagDetailsResponse, FavoriteDagData, FavoriteDagResponse, UnfavoriteDagData, UnfavoriteDagResponse, GetDagTagsData, GetDagTagsResponse, GetDagsUiData, GetDagsUiResponse, GetLatestRunInfoData, GetLatestRunInfoResponse, GetDagRunStateCountsUiData, GetDagRunStateCountsUiResponse, GetEventLogData, GetEventLogResponse, GetEventLogsData, GetEventLogsResponse, GetExtraLinksData, GetExtraLinksResponse, GetTaskInstanceData, GetTaskInstanceResponse, PatchTaskInstanceData, PatchTaskInstanceResponse, DeleteTaskInstanceData, DeleteTaskInstanceResponse, GetMappedTaskInstancesData, GetMappedTaskInstancesResponse, GetTaskInstanceDependenciesByMapIndexData, GetTaskInstanceDependenciesByMapIndexResponse, GetTaskInstanceDependenciesData, GetTaskInstanceDependenciesResponse, GetTaskInstanceTriesData, GetTaskInstanceTriesResponse, GetMappedTaskInstanceTriesData, GetMappedTaskInstanceTriesResponse, GetMappedTaskInstanceData, GetMappedTaskInstanceResponse, PatchTaskInstanceByMapIndexData, PatchTaskInstanceByMapIndexResponse, GetTaskInstancesData, GetTaskInstancesResponse, BulkTaskInstancesData, BulkTaskInstancesResponse, GetTaskInstancesBatchData, GetTaskInstancesBatchResponse, GetTaskInstanceTryDetailsData, GetTaskInstanceTryDetailsResponse, GetMappedTaskInstanceTryDetailsData, GetMappedTaskInstanceTryDetailsResponse, PostClearTaskInstancesData, PostClearTaskInstancesResponse, PatchTaskGroupInstancesData, PatchTaskGroupInstancesResponse, PatchTaskGroupInstancesDryRunData, PatchTaskGroupInstancesDryRunResponse, PatchTaskInstanceDryRunByMapIndexData, PatchTaskInstanceDryRunByMapIndexResponse, PatchTaskInstanceDryRunData, PatchTaskInstanceDryRunResponse, GetLogData, GetLogResponse, GetExternalLogUrlData, GetExternalLogUrlResponse, UpdateHitlDetailData, UpdateHitlDetailResponse, GetHitlDetailData, GetHitlDetailResponse, GetHitlDetailTryDetailData, GetHitlDetailTryDetailResponse, GetHitlDetailsData, GetHitlDetailsResponse, GetImportErrorData, GetImportErrorResponse, GetImportErrorsData, GetImportErrorsResponse, GetJobsData, GetJobsResponse, GetPluginsData, GetPluginsResponse, ImportErrorsResponse, DeletePoolData, DeletePoolResponse, GetPoolData, GetPoolResponse, PatchPoolData, PatchPoolResponse, GetPoolsData, GetPoolsResponse, PostPoolData, PostPoolResponse, BulkPoolsData, BulkPoolsResponse, GetProvidersData, GetProvidersResponse, ListAssetStateStoreData, ListAssetStateStoreResponse, ClearAssetStateStoreData, ClearAssetStateStoreResponse, GetAssetStateStoreData, GetAssetStateStoreResponse, SetAssetStateStoreData, SetAssetStateStoreResponse, DeleteAssetStateStoreData, DeleteAssetStateStoreResponse, ListTaskStateStoreData, ListTaskStateStoreResponse, ClearTaskStateStoreData, ClearTaskStateStoreResponse, GetTaskStateStoreData, GetTaskStateStoreResponse, SetTaskStateStoreData, SetTaskStateStoreResponse, PatchTaskStateStoreData, PatchTaskStateStoreResponse, DeleteTaskStateStoreData, DeleteTaskStateStoreResponse, GetXcomEntryData, GetXcomEntryResponse, UpdateXcomEntryData, UpdateXcomEntryResponse, DeleteXcomEntryData, DeleteXcomEntryResponse, GetXcomEntriesData, GetXcomEntriesResponse, CreateXcomEntryData, CreateXcomEntryResponse, GetTasksData, GetTasksResponse, GetTaskData, GetTaskResponse, DeleteVariableData, DeleteVariableResponse, GetVariableData, GetVariableResponse, PatchVariableData, PatchVariableResponse, GetVariablesData, GetVariablesResponse, PostVariableData, PostVariableResponse, BulkVariablesData, BulkVariablesResponse, ReparseDagFileData, ReparseDagFileResponse, GetDagVersionData, GetDagVersionResponse, GetDagVersionsData, GetDagVersionsResponse, GetHealthResponse, GetVersionResponse, LoginData, LoginResponse, LogoutResponse, GetAuthMenusResponse, GetCurrentUserInfoResponse, GenerateTokenData, GenerateTokenResponse2, GetPartitionedDagRunsData, GetPartitionedDagRunsResponse, GetPendingPartitionedDagRunData, GetPendingPartitionedDagRunResponse, GetDependenciesData, GetDependenciesResponse, HistoricalMetricsData, HistoricalMetricsResponse, DagStatsResponse2, GetDeadlinesData, GetDeadlinesResponse, GetDagDeadlineAlertsData, GetDagDeadlineAlertsResponse, StructureDataData, StructureDataResponse2, GetDagStructureData, GetDagStructureResponse, GetGridRunsData, GetGridRunsResponse, GetGridTiSummariesStreamData, GetGridTiSummariesStreamResponse, GetGanttDataData, GetGanttDataResponse, GetCalendarData, GetCalendarResponse, GetCalendarDeadlinesData, GetCalendarDeadlinesResponse, ListTeamsData, ListTeamsResponse } from './types.gen'; +import type { GetAssetsData, GetAssetsResponse, GetAssetAliasesData, GetAssetAliasesResponse, GetAssetAliasData, GetAssetAliasResponse, GetAssetEventsData, GetAssetEventsResponse, CreateAssetEventData, CreateAssetEventResponse, MaterializeAssetData, MaterializeAssetResponse, GetAssetQueuedEventsData, GetAssetQueuedEventsResponse, DeleteAssetQueuedEventsData, DeleteAssetQueuedEventsResponse, GetAssetData, GetAssetResponse, GetDagAssetQueuedEventsData, GetDagAssetQueuedEventsResponse, DeleteDagAssetQueuedEventsData, DeleteDagAssetQueuedEventsResponse, GetDagAssetQueuedEventData, GetDagAssetQueuedEventResponse, DeleteDagAssetQueuedEventData, DeleteDagAssetQueuedEventResponse, NextRunAssetsData, NextRunAssetsResponse2, ListBackfillsData, ListBackfillsResponse, CreateBackfillData, CreateBackfillResponse, GetBackfillData, GetBackfillResponse, ListBackfillDagRunsData, ListBackfillDagRunsResponse, PauseBackfillData, PauseBackfillResponse, UnpauseBackfillData, UnpauseBackfillResponse, CancelBackfillData, CancelBackfillResponse, CreateBackfillDryRunData, CreateBackfillDryRunResponse, ListBackfillsUiData, ListBackfillsUiResponse, DeleteConnectionData, DeleteConnectionResponse, GetConnectionData, GetConnectionResponse, PatchConnectionData, PatchConnectionResponse, GetConnectionTestData, GetConnectionTestResponse, EnqueueConnectionTestData, EnqueueConnectionTestResponse, GetConnectionsData, GetConnectionsResponse, PostConnectionData, PostConnectionResponse, BulkConnectionsData, BulkConnectionsResponse, TestConnectionData, TestConnectionResponse, CreateDefaultConnectionsResponse, HookMetaDataResponse, GetDagRunData, GetDagRunResponse, DeleteDagRunData, DeleteDagRunResponse, PatchDagRunData, PatchDagRunResponse, BulkDagRunsData, BulkDagRunsResponse, GetDagRunsData, GetDagRunsResponse, TriggerDagRunData, TriggerDagRunResponse, GetUpstreamAssetEventsData, GetUpstreamAssetEventsResponse, ClearDagRunData, ClearDagRunResponse, WaitDagRunUntilFinishedData, WaitDagRunUntilFinishedResponse, GetListDagRunsBatchData, GetListDagRunsBatchResponse, ClearDagRunsData, ClearDagRunsResponse, ClearDagRunPartitionsData, ClearDagRunPartitionsResponse, GetDagRunStatsData, GetDagRunStatsResponse, GetDagSourceData, GetDagSourceResponse, GetDagStatsData, GetDagStatsResponse, GetConfigData, GetConfigResponse, GetConfigValueData, GetConfigValueResponse, GetConfigsResponse, ListDagWarningsData, ListDagWarningsResponse, GetDagsData, GetDagsResponse, PatchDagsData, PatchDagsResponse, GetDagData, GetDagResponse, PatchDagData, PatchDagResponse, DeleteDagData, DeleteDagResponse, GetDagDetailsData, GetDagDetailsResponse, FavoriteDagData, FavoriteDagResponse, UnfavoriteDagData, UnfavoriteDagResponse, GetDagTagsData, GetDagTagsResponse, GetDagsUiData, GetDagsUiResponse, GetDagFoldersResponse, GetLatestRunInfoData, GetLatestRunInfoResponse, GetDagRunStateCountsUiData, GetDagRunStateCountsUiResponse, GetEventLogData, GetEventLogResponse, GetEventLogsData, GetEventLogsResponse, GetExtraLinksData, GetExtraLinksResponse, GetTaskInstanceData, GetTaskInstanceResponse, PatchTaskInstanceData, PatchTaskInstanceResponse, DeleteTaskInstanceData, DeleteTaskInstanceResponse, GetMappedTaskInstancesData, GetMappedTaskInstancesResponse, GetTaskInstanceDependenciesByMapIndexData, GetTaskInstanceDependenciesByMapIndexResponse, GetTaskInstanceDependenciesData, GetTaskInstanceDependenciesResponse, GetTaskInstanceTriesData, GetTaskInstanceTriesResponse, GetMappedTaskInstanceTriesData, GetMappedTaskInstanceTriesResponse, GetMappedTaskInstanceData, GetMappedTaskInstanceResponse, PatchTaskInstanceByMapIndexData, PatchTaskInstanceByMapIndexResponse, GetTaskInstancesData, GetTaskInstancesResponse, BulkTaskInstancesData, BulkTaskInstancesResponse, GetTaskInstancesBatchData, GetTaskInstancesBatchResponse, GetTaskInstanceTryDetailsData, GetTaskInstanceTryDetailsResponse, GetMappedTaskInstanceTryDetailsData, GetMappedTaskInstanceTryDetailsResponse, PostClearTaskInstancesData, PostClearTaskInstancesResponse, PatchTaskGroupInstancesData, PatchTaskGroupInstancesResponse, PatchTaskGroupInstancesDryRunData, PatchTaskGroupInstancesDryRunResponse, PatchTaskInstanceDryRunByMapIndexData, PatchTaskInstanceDryRunByMapIndexResponse, PatchTaskInstanceDryRunData, PatchTaskInstanceDryRunResponse, GetLogData, GetLogResponse, GetExternalLogUrlData, GetExternalLogUrlResponse, UpdateHitlDetailData, UpdateHitlDetailResponse, GetHitlDetailData, GetHitlDetailResponse, GetHitlDetailTryDetailData, GetHitlDetailTryDetailResponse, GetHitlDetailsData, GetHitlDetailsResponse, GetImportErrorData, GetImportErrorResponse, GetImportErrorsData, GetImportErrorsResponse, GetJobsData, GetJobsResponse, GetPluginsData, GetPluginsResponse, ImportErrorsResponse, DeletePoolData, DeletePoolResponse, GetPoolData, GetPoolResponse, PatchPoolData, PatchPoolResponse, GetPoolsData, GetPoolsResponse, PostPoolData, PostPoolResponse, BulkPoolsData, BulkPoolsResponse, GetProvidersData, GetProvidersResponse, ListAssetStateStoreData, ListAssetStateStoreResponse, ClearAssetStateStoreData, ClearAssetStateStoreResponse, GetAssetStateStoreData, GetAssetStateStoreResponse, SetAssetStateStoreData, SetAssetStateStoreResponse, DeleteAssetStateStoreData, DeleteAssetStateStoreResponse, ListTaskStateStoreData, ListTaskStateStoreResponse, ClearTaskStateStoreData, ClearTaskStateStoreResponse, GetTaskStateStoreData, GetTaskStateStoreResponse, SetTaskStateStoreData, SetTaskStateStoreResponse, PatchTaskStateStoreData, PatchTaskStateStoreResponse, DeleteTaskStateStoreData, DeleteTaskStateStoreResponse, GetXcomEntryData, GetXcomEntryResponse, UpdateXcomEntryData, UpdateXcomEntryResponse, DeleteXcomEntryData, DeleteXcomEntryResponse, GetXcomEntriesData, GetXcomEntriesResponse, CreateXcomEntryData, CreateXcomEntryResponse, GetTasksData, GetTasksResponse, GetTaskData, GetTaskResponse, DeleteVariableData, DeleteVariableResponse, GetVariableData, GetVariableResponse, PatchVariableData, PatchVariableResponse, GetVariablesData, GetVariablesResponse, PostVariableData, PostVariableResponse, BulkVariablesData, BulkVariablesResponse, ReparseDagFileData, ReparseDagFileResponse, GetDagVersionData, GetDagVersionResponse, GetDagVersionsData, GetDagVersionsResponse, GetHealthResponse, GetVersionResponse, LoginData, LoginResponse, LogoutResponse, GetAuthMenusResponse, GetCurrentUserInfoResponse, GenerateTokenData, GenerateTokenResponse2, GetPartitionedDagRunsData, GetPartitionedDagRunsResponse, GetPendingPartitionedDagRunData, GetPendingPartitionedDagRunResponse, GetDependenciesData, GetDependenciesResponse, HistoricalMetricsData, HistoricalMetricsResponse, DagStatsResponse2, GetDeadlinesData, GetDeadlinesResponse, GetDagDeadlineAlertsData, GetDagDeadlineAlertsResponse, StructureDataData, StructureDataResponse2, GetDagStructureData, GetDagStructureResponse, GetGridRunsData, GetGridRunsResponse, GetGridTiSummariesStreamData, GetGridTiSummariesStreamResponse, GetGanttDataData, GetGanttDataResponse, GetCalendarData, GetCalendarResponse, GetCalendarDeadlinesData, GetCalendarDeadlinesResponse, ListTeamsData, ListTeamsResponse } from './types.gen'; export class AssetService { /** @@ -1967,6 +1967,7 @@ export class DagService { * @param data.dagRunState Filter Dags that have any DagRun in the given state. Only ``queued`` and ``running`` are supported. * @param data.bundleName * @param data.bundleVersion + * @param data.relativeFilelocPrefix Filter Dags by the folder (directory of ``relative_fileloc``) they live in. Matches the given folder and all of its subfolders. * @param data.orderBy Attributes to order by, multi criteria sort is supported. Prefix with `-` for descending order. Supported attributes: `dag_id, dag_display_name, next_dagrun, state, start_date, last_run_state, last_run_start_date, last_run_run_after` * @param data.isFavorite * @param data.hasAssetSchedule Filter Dags with asset-based scheduling @@ -1999,6 +2000,7 @@ export class DagService { dag_run_state: data.dagRunState, bundle_name: data.bundleName, bundle_version: data.bundleVersion, + relative_fileloc_prefix: data.relativeFilelocPrefix, order_by: data.orderBy, is_favorite: data.isFavorite, has_asset_schedule: data.hasAssetSchedule, @@ -2011,6 +2013,27 @@ export class DagService { }); } + /** + * Get Dag Folders + * Get the distinct folders the readable Dags live in, scoped to their bundle. + * + * A folder is the directory part of a Dag's ``relative_fileloc`` (relative to its + * bundle root). Because ``relative_fileloc`` is relative to each bundle, the same + * path can exist in several bundles, so every folder is paired with its bundle + * name to keep them apart. Dags located directly at the bundle root have no folder + * and are not represented here. The result powers the folder navigation tree in + * the UI, which reconstructs the hierarchy by splitting each path on ``/`` and + * groups it under its bundle when more than one bundle is present. + * @returns DagFolderCollectionResponse Successful Response + * @throws ApiError + */ + public static getDagFolders(): CancelablePromise { + return __request(OpenAPI, { + method: 'GET', + url: '/ui/dags/folders' + }); + } + /** * Get Latest Run Info * Get latest run. diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index 38bc1662831dc..e55b95a495f38 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -2397,6 +2397,22 @@ export type DAGsRunStateCountsCollectionResponse = { state_count_limit: number; }; +/** + * Collection of distinct Dag folders, each scoped to the bundle it belongs to. + */ +export type DagFolderCollectionResponse = { + folders: Array; + total_entries: number; +}; + +/** + * A distinct Dag folder (directory of ``relative_fileloc``) within a bundle. + */ +export type DagFolderResponse = { + bundle_name: string; + folder: string; +}; + /** * DAG Run statistics serializer for responses. */ @@ -3565,6 +3581,10 @@ export type GetDagsUiData = { orderBy?: Array<(string)>; owners?: Array<(string)>; paused?: boolean | null; + /** + * Filter Dags by the folder (directory of ``relative_fileloc``) they live in. Matches the given folder and all of its subfolders. + */ + relativeFilelocPrefix?: string | null; tags?: Array<(string)>; tagsMatchMode?: 'any' | 'all' | null; teams?: Array<(string)>; @@ -3572,6 +3592,8 @@ export type GetDagsUiData = { export type GetDagsUiResponse = DAGWithLatestDagRunsCollectionResponse; +export type GetDagFoldersResponse = DagFolderCollectionResponse; + export type GetLatestRunInfoData = { dagId: string; }; @@ -6470,6 +6492,16 @@ export type $OpenApiTs = { }; }; }; + '/ui/dags/folders': { + get: { + res: { + /** + * Successful Response + */ + 200: DagFolderCollectionResponse; + }; + }; + }; '/ui/dags/{dag_id}/latest_run': { get: { req: GetLatestRunInfoData; diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/en/dags.json b/airflow-core/src/airflow/ui/public/i18n/locales/en/dags.json index 78b8681b423a5..a22ca2353ba00 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/en/dags.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/en/dags.json @@ -24,6 +24,11 @@ }, "runIdPatternFilter": "Search Dag Runs" }, + "folders": { + "all": "All Dags", + "empty": "No folders", + "title": "Folders" + }, "ownerLink": "Owner link for {{owner}}", "runAndTaskActions": { "affectedTasks": { diff --git a/airflow-core/src/airflow/ui/src/constants/searchParams.ts b/airflow-core/src/airflow/ui/src/constants/searchParams.ts index eee952f020786..e1009cfb72d70 100644 --- a/airflow-core/src/airflow/ui/src/constants/searchParams.ts +++ b/airflow-core/src/airflow/ui/src/constants/searchParams.ts @@ -28,7 +28,9 @@ export enum SearchParamsKeys { CREATED_AT_LTE = "created_at_lte", CREATED_AT_RANGE = "created_at_range", CURSOR = "cursor", + DAG_BUNDLE = "dag_bundle", DAG_DISPLAY_NAME_PATTERN = "dag_display_name_pattern", + DAG_FOLDER = "dag_folder", DAG_ID = "dag_id", DAG_ID_PATTERN = "dag_id_pattern", DAG_RUN_STATE = "dag_run_state", diff --git a/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/DagFolderTree.test.tsx b/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/DagFolderTree.test.tsx new file mode 100644 index 0000000000000..82eea9ad9ca47 --- /dev/null +++ b/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/DagFolderTree.test.tsx @@ -0,0 +1,230 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import "@testing-library/jest-dom"; +import { fireEvent, render, screen } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; + +import type { DagFolderResponse } from "openapi/requests/types.gen"; +import { BaseWrapper } from "src/utils/Wrapper"; + +import { DagFolderTree } from "./DagFolderTree"; + +const SINGLE_BUNDLE: Array = [ + { bundle_name: "dags-folder", folder: "team_a/etl" }, + { bundle_name: "dags-folder", folder: "team_a/report" }, + { bundle_name: "dags-folder", folder: "team_b/ml" }, +]; + +// Both bundles reuse the ``team_a/etl`` path to check they stay separate. +const MULTI_BUNDLE: Array = [ + { bundle_name: "analytics", folder: "team_a/etl" }, + { bundle_name: "ml", folder: "team_a/etl" }, + { bundle_name: "ml", folder: "features" }, +]; + +describe("DagFolderTree (single bundle)", () => { + it("renders the top-level folders and an 'All Dags' entry, without a bundle level", () => { + render( + , + { wrapper: BaseWrapper }, + ); + + expect(screen.getByText("folders.all")).toBeInTheDocument(); + expect(screen.getByText("team_a")).toBeInTheDocument(); + expect(screen.getByText("team_b")).toBeInTheDocument(); + expect(screen.queryByText("dags-folder")).not.toBeInTheDocument(); + }); + + it("keeps sub-folders collapsed until expanded", () => { + render( + , + { wrapper: BaseWrapper }, + ); + + expect(screen.queryByText("etl")).not.toBeInTheDocument(); + + fireEvent.click(screen.getAllByLabelText("Expand")[0] as HTMLElement); + + expect(screen.getByText("etl")).toBeInTheDocument(); + expect(screen.getByText("report")).toBeInTheDocument(); + }); + + it("auto-expands the ancestors of the selected folder", () => { + render( + , + { wrapper: BaseWrapper }, + ); + + expect(screen.getByText("etl")).toBeInTheDocument(); + }); + + it("calls onSelectFolder with the folder path (no bundle) when a folder is clicked", () => { + const onSelectFolder = vi.fn(); + + render( + , + { wrapper: BaseWrapper }, + ); + + fireEvent.click(screen.getByText("team_b")); + + expect(onSelectFolder).toHaveBeenCalledWith({ bundleName: undefined, folder: "team_b" }); + }); + + it("clears the selection when 'All Dags' is clicked", () => { + const onSelectFolder = vi.fn(); + + render( + , + { wrapper: BaseWrapper }, + ); + + fireEvent.click(screen.getByText("folders.all")); + + expect(onSelectFolder).toHaveBeenCalledWith({ bundleName: undefined, folder: undefined }); + }); + + it("does not select the folder when toggling its expander", () => { + const onSelectFolder = vi.fn(); + + render( + , + { wrapper: BaseWrapper }, + ); + + fireEvent.click(screen.getAllByLabelText("Expand")[0] as HTMLElement); + + expect(onSelectFolder).not.toHaveBeenCalled(); + }); + + it("shows an empty message when there are no folders", () => { + render( + , + { wrapper: BaseWrapper }, + ); + + expect(screen.getByText("folders.empty")).toBeInTheDocument(); + }); +}); + +describe("DagFolderTree (multiple bundles)", () => { + it("renders bundles as top-level entries", () => { + render( + , + { wrapper: BaseWrapper }, + ); + + expect(screen.getByText("analytics")).toBeInTheDocument(); + expect(screen.getByText("ml")).toBeInTheDocument(); + // Folders stay hidden until a bundle is expanded. + expect(screen.queryByText("features")).not.toBeInTheDocument(); + }); + + it("calls onSelectFolder with the bundle (no folder) when a bundle is clicked", () => { + const onSelectFolder = vi.fn(); + + render( + , + { wrapper: BaseWrapper }, + ); + + fireEvent.click(screen.getByText("analytics")); + + expect(onSelectFolder).toHaveBeenCalledWith({ bundleName: "analytics", folder: undefined }); + }); + + it("keeps the same folder path separate under each bundle", () => { + const onSelectFolder = vi.fn(); + + render( + , + { wrapper: BaseWrapper }, + ); + + // Expand the "ml" bundle (second one alphabetically) and select its leaf folder. + fireEvent.click(screen.getAllByLabelText("Expand")[1] as HTMLElement); + fireEvent.click(screen.getByText("features")); + + expect(onSelectFolder).toHaveBeenCalledWith({ bundleName: "ml", folder: "features" }); + }); + + it("auto-expands the selected bundle down to the selected folder", () => { + render( + , + { wrapper: BaseWrapper }, + ); + + expect(screen.getByText("etl")).toBeInTheDocument(); + }); +}); diff --git a/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/DagFolderTree.tsx b/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/DagFolderTree.tsx new file mode 100644 index 0000000000000..5577fc5ba9157 --- /dev/null +++ b/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/DagFolderTree.tsx @@ -0,0 +1,320 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Box, Flex, Heading, Skeleton, Text, VStack } from "@chakra-ui/react"; +import { useState } from "react"; +import { useTranslation } from "react-i18next"; +import { FiBox, FiChevronDown, FiChevronRight, FiFolder } from "react-icons/fi"; + +import type { DagFolderResponse } from "openapi/requests/types.gen"; + +import { groupFoldersByBundle, type BundleFolders, type FolderNode } from "./buildFolderTree"; + +export type FolderSelection = { + readonly bundleName: string | undefined; + readonly folder: string | undefined; +}; + +type Props = { + readonly folders: ReadonlyArray; + readonly isLoading?: boolean; + readonly onSelectFolder: (selection: FolderSelection) => void; + readonly selectedBundle: string | undefined; + readonly selectedFolder: string | undefined; +}; + +const folderKey = (bundleName: string | undefined, path: string) => `folder:${bundleName ?? ""}:${path}`; +const bundleKey = (bundleName: string) => `bundle:${bundleName}`; + +// Ancestor folder paths of the selected folder, so the tree opens to reveal the selection. +const ancestorPaths = (folder: string | undefined): Array => { + if (folder === undefined || folder === "") { + return []; + } + + const segments = folder.split("/"); + + return segments.map((_, index) => segments.slice(0, index + 1).join("/")); +}; + +const initialExpanded = ( + bundleName: string | undefined, + selectedFolder: string | undefined, + isMultiBundle: boolean, +): Array => { + const keys = ancestorPaths(selectedFolder).map((path) => folderKey(bundleName, path)); + + if (isMultiBundle && bundleName !== undefined) { + keys.push(bundleKey(bundleName)); + } + + return keys; +}; + +type FolderRowProps = { + readonly bundleName: string | undefined; + readonly depthOffset: number; + readonly expanded: Set; + readonly node: FolderNode; + readonly onSelectFolder: (selection: FolderSelection) => void; + readonly onToggle: (key: string) => void; + readonly selectedBundle: string | undefined; + readonly selectedFolder: string | undefined; +}; + +const FolderRow = ({ + bundleName, + depthOffset, + expanded, + node, + onSelectFolder, + onToggle, + selectedBundle, + selectedFolder, +}: FolderRowProps) => { + const hasChildren = node.children.length > 0; + const key = folderKey(bundleName, node.path); + const isExpanded = expanded.has(key); + const isSelected = selectedBundle === bundleName && selectedFolder === node.path; + const depth = node.path.split("/").length - 1 + depthOffset; + + return ( + + onSelectFolder({ bundleName, folder: node.path })} + pl={`${depth * 16 + 4}px`} + py={1} + > + { + event.stopPropagation(); + if (hasChildren) { + onToggle(key); + } + }} + visibility={hasChildren ? "visible" : "hidden"} + > + {isExpanded ? : } + + + + + + {node.name} + + + {hasChildren && isExpanded ? ( + + {node.children.map((child) => ( + + ))} + + ) : undefined} + + ); +}; + +type BundleRowProps = { + readonly bundle: BundleFolders; + readonly expanded: Set; + readonly onSelectFolder: (selection: FolderSelection) => void; + readonly onToggle: (key: string) => void; + readonly selectedBundle: string | undefined; + readonly selectedFolder: string | undefined; +}; + +const BundleRow = ({ + bundle, + expanded, + onSelectFolder, + onToggle, + selectedBundle, + selectedFolder, +}: BundleRowProps) => { + const key = bundleKey(bundle.bundleName); + const isExpanded = expanded.has(key); + const isSelected = selectedBundle === bundle.bundleName && selectedFolder === undefined; + const hasChildren = bundle.tree.length > 0; + + return ( + + onSelectFolder({ bundleName: bundle.bundleName, folder: undefined })} + pl="4px" + py={1} + > + { + event.stopPropagation(); + if (hasChildren) { + onToggle(key); + } + }} + visibility={hasChildren ? "visible" : "hidden"} + > + {isExpanded ? : } + + + + + + {bundle.bundleName} + + + {hasChildren && isExpanded ? ( + + {bundle.tree.map((node) => ( + + ))} + + ) : undefined} + + ); +}; + +export const DagFolderTree = ({ + folders, + isLoading = false, + onSelectFolder, + selectedBundle, + selectedFolder, +}: Props) => { + const { t: translate } = useTranslation("dags"); + const bundles = groupFoldersByBundle(folders); + const isMultiBundle = bundles.length > 1; + + const [expanded, setExpanded] = useState>( + () => new Set(initialExpanded(isMultiBundle ? selectedBundle : undefined, selectedFolder, isMultiBundle)), + ); + + const handleToggle = (key: string) => { + setExpanded((previous) => { + const next = new Set(previous); + + if (next.has(key)) { + next.delete(key); + } else { + next.add(key); + } + + return next; + }); + }; + + const isAllSelected = selectedBundle === undefined && selectedFolder === undefined; + + return ( + + + {translate("folders.title")} + + {isLoading ? ( + + + + + + ) : ( + + onSelectFolder({ bundleName: undefined, folder: undefined })} + pl="4px" + py={1} + > + {translate("folders.all")} + + {bundles.length === 0 ? ( + + {translate("folders.empty")} + + ) : isMultiBundle ? ( + bundles.map((bundle) => ( + + )) + ) : ( + bundles[0]?.tree.map((node) => ( + + )) + )} + + )} + + ); +}; diff --git a/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/buildFolderTree.test.ts b/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/buildFolderTree.test.ts new file mode 100644 index 0000000000000..9521f9775515b --- /dev/null +++ b/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/buildFolderTree.test.ts @@ -0,0 +1,119 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { describe, expect, it } from "vitest"; + +import { buildFolderTree, groupFoldersByBundle } from "./buildFolderTree"; + +describe("buildFolderTree", () => { + it("returns an empty array for no folders", () => { + expect(buildFolderTree([])).toEqual([]); + }); + + it("builds single-level roots", () => { + const tree = buildFolderTree(["team_a", "team_b"]); + + expect(tree.map((node) => node.path)).toEqual(["team_a", "team_b"]); + expect(tree.every((node) => node.children.length === 0)).toBe(true); + }); + + it("nests sub-folders and keeps full paths", () => { + const tree = buildFolderTree(["team_a/etl", "team_b/ml"]); + + expect(tree.map((node) => node.path)).toEqual(["team_a", "team_b"]); + + const [teamA] = tree; + + expect(teamA?.name).toBe("team_a"); + expect(teamA?.children).toHaveLength(1); + expect(teamA?.children[0]?.name).toBe("etl"); + expect(teamA?.children[0]?.path).toBe("team_a/etl"); + }); + + it("synthesizes intermediate folders that contain no Dag of their own", () => { + const tree = buildFolderTree(["team_a/etl/extract"]); + + expect(tree).toHaveLength(1); + expect(tree[0]?.path).toBe("team_a"); + expect(tree[0]?.children[0]?.path).toBe("team_a/etl"); + expect(tree[0]?.children[0]?.children[0]?.path).toBe("team_a/etl/extract"); + }); + + it("merges a folder that is both a leaf and a parent", () => { + const tree = buildFolderTree(["team_a", "team_a/etl"]); + + expect(tree).toHaveLength(1); + expect(tree[0]?.path).toBe("team_a"); + expect(tree[0]?.children.map((node) => node.path)).toEqual(["team_a/etl"]); + }); + + it("deduplicates repeated folders", () => { + const tree = buildFolderTree(["team_a/etl", "team_a/etl"]); + + expect(tree).toHaveLength(1); + expect(tree[0]?.children).toHaveLength(1); + }); + + it("sorts siblings alphabetically at every level", () => { + const tree = buildFolderTree(["team_b/zeta", "team_b/alpha", "team_a"]); + + expect(tree.map((node) => node.name)).toEqual(["team_a", "team_b"]); + const teamB = tree.find((node) => node.name === "team_b"); + + expect(teamB?.children.map((node) => node.name)).toEqual(["alpha", "zeta"]); + }); +}); + +describe("groupFoldersByBundle", () => { + it("returns an empty array for no folders", () => { + expect(groupFoldersByBundle([])).toEqual([]); + }); + + it("groups folders under their bundle and sorts bundles alphabetically", () => { + const bundles = groupFoldersByBundle([ + { bundle_name: "ml", folder: "features" }, + { bundle_name: "analytics", folder: "team_a/etl" }, + ]); + + expect(bundles.map((bundle) => bundle.bundleName)).toEqual(["analytics", "ml"]); + }); + + it("keeps the same folder path separate per bundle", () => { + const bundles = groupFoldersByBundle([ + { bundle_name: "analytics", folder: "team_a/etl" }, + { bundle_name: "ml", folder: "team_a/etl" }, + ]); + + const analytics = bundles.find((bundle) => bundle.bundleName === "analytics"); + const ml = bundles.find((bundle) => bundle.bundleName === "ml"); + + expect(analytics?.tree[0]?.children[0]?.path).toBe("team_a/etl"); + expect(ml?.tree[0]?.children[0]?.path).toBe("team_a/etl"); + }); + + it("builds a full folder tree within each bundle", () => { + const bundles = groupFoldersByBundle([ + { bundle_name: "b1", folder: "team_a/etl" }, + { bundle_name: "b1", folder: "team_a/report" }, + ]); + + expect(bundles).toHaveLength(1); + expect(bundles[0]?.tree[0]?.path).toBe("team_a"); + expect(bundles[0]?.tree[0]?.children.map((node) => node.name)).toEqual(["etl", "report"]); + }); +}); diff --git a/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/buildFolderTree.ts b/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/buildFolderTree.ts new file mode 100644 index 0000000000000..8ae5bce668cca --- /dev/null +++ b/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/buildFolderTree.ts @@ -0,0 +1,104 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +export type FolderNode = { + /** Direct child folders, keyed alphabetically by their display name. */ + readonly children: Array; + /** Last path segment, shown in the tree (e.g. ``etl``). */ + readonly name: string; + /** Full folder path from the bundle root (e.g. ``team_a/etl``), used as the filter value. */ + readonly path: string; +}; + +/** + * Build a nested folder tree from a flat list of folder paths. + * + * The backend only returns the directory of each Dag file (leaf folders), so intermediate + * folders that contain no Dag of their own — but do contain sub-folders — are synthesized + * here. Given ``["team_a/etl", "team_b/ml"]`` the ``team_a``/``team_b`` nodes are created even + * though no Dag lives directly in them. + * + * Children are sorted alphabetically at every level for a stable, predictable rendering. + */ +export const buildFolderTree = (folders: ReadonlyArray): Array => { + type MutableNode = { children: Map; name: string; path: string }; + + const roots = new Map(); + + for (const folder of folders) { + const segments = folder.split("/").filter((segment) => segment !== ""); + + let level = roots; + let prefix = ""; + + for (const segment of segments) { + prefix = prefix === "" ? segment : `${prefix}/${segment}`; + + let node = level.get(segment); + + if (node === undefined) { + node = { children: new Map(), name: segment, path: prefix }; + level.set(segment, node); + } + + level = node.children; + } + } + + const toSortedNodes = (level: Map): Array => + [...level.values()] + .sort((left, right) => left.name.localeCompare(right.name)) + .map((node) => ({ + children: toSortedNodes(node.children), + name: node.name, + path: node.path, + })); + + return toSortedNodes(roots); +}; + +export type BundleFolders = { + /** The bundle these folders belong to. */ + readonly bundleName: string; + /** Folder tree built from this bundle's folder paths. */ + readonly tree: Array; +}; + +/** + * Group folders by their bundle, then build a folder tree per bundle. + * + * Because ``relative_fileloc`` is relative to each bundle root, the same folder path + * can exist in several bundles; grouping first keeps them separate. Bundles are sorted + * alphabetically for stable rendering. + */ +export const groupFoldersByBundle = ( + folders: ReadonlyArray<{ readonly bundle_name: string; readonly folder: string }>, +): Array => { + const pathsByBundle = new Map>(); + + for (const { bundle_name: bundleName, folder } of folders) { + const paths = pathsByBundle.get(bundleName) ?? []; + + paths.push(folder); + pathsByBundle.set(bundleName, paths); + } + + return [...pathsByBundle.entries()] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([bundleName, paths]) => ({ bundleName, tree: buildFolderTree(paths) })); +}; diff --git a/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/index.ts b/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/index.ts new file mode 100644 index 0000000000000..6dd75a1d75632 --- /dev/null +++ b/airflow-core/src/airflow/ui/src/pages/DagsList/DagFolderTree/index.ts @@ -0,0 +1,19 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +export { DagFolderTree, type FolderSelection } from "./DagFolderTree"; diff --git a/airflow-core/src/airflow/ui/src/pages/DagsList/DagsList.tsx b/airflow-core/src/airflow/ui/src/pages/DagsList/DagsList.tsx index f82a9e90c428a..7a192576a9491 100644 --- a/airflow-core/src/airflow/ui/src/pages/DagsList/DagsList.tsx +++ b/airflow-core/src/airflow/ui/src/pages/DagsList/DagsList.tsx @@ -16,7 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -import { Heading, HStack, Skeleton, VStack, type SelectValueChangeDetails, Box } from "@chakra-ui/react"; +import { + Heading, + HStack, + Skeleton, + VStack, + type SelectValueChangeDetails, + Box, + Flex, +} from "@chakra-ui/react"; import type { ColumnDef } from "@tanstack/react-table"; import { useTranslation } from "react-i18next"; import { useSearchParams } from "react-router-dom"; @@ -40,12 +48,14 @@ import { SearchParamsKeys, type SearchParamsKeysType } from "src/constants/searc import { useAdvancedSearch } from "src/hooks/useAdvancedSearch"; import { DagsLayout } from "src/layouts/DagsLayout"; import { useConfig } from "src/queries/useConfig"; +import { useDagFolders } from "src/queries/useDagFolders"; import { useDagRunStateCounts } from "src/queries/useDagRunStateCounts"; import { useDags } from "src/queries/useDags"; import { useDocumentTitle } from "src/utils"; import { DagImportErrors } from "../Dashboard/Stats/DagImportErrors"; import { DagCard } from "./DagCard"; +import { DagFolderTree, type FolderSelection } from "./DagFolderTree"; import { DagRunStateCounts } from "./DagRunStateCounts"; import { DagTags } from "./DagTags"; import { DagsFilters } from "./DagsFilters"; @@ -212,6 +222,8 @@ const createColumns = ( ]; const { + DAG_BUNDLE, + DAG_FOLDER, DAG_RUN_STATE, FAVORITE, LAST_DAG_RUN_STATE, @@ -259,6 +271,14 @@ export const DagsList = () => { const pendingReviews = searchParams.get(NEEDS_REVIEW); const owners = searchParams.getAll(OWNERS); const teams = searchParams.getAll(TEAMS); + const selectedFolder = searchParams.get(DAG_FOLDER) ?? undefined; + const selectedBundle = searchParams.get(DAG_BUNDLE) ?? undefined; + + const { folders, isLoading: foldersLoading } = useDagFolders(); + // Keep the panel out of the way for flat deployments (all Dags at the bundle root). Still show it + // while loading, or when a folder/bundle is selected so the user can always navigate back to "All Dags". + const showFolderTree = + foldersLoading || folders.length > 0 || Boolean(selectedFolder) || Boolean(selectedBundle); const { setTableURLState, tableURLState } = useTableURLState(); @@ -283,6 +303,25 @@ export const DagsList = () => { setSearchParams(searchParams); }; + const handleFolderChange = ({ bundleName, folder }: FolderSelection) => { + setTableURLState({ + pagination: { ...pagination, pageIndex: 0 }, + sorting, + }); + if (folder === undefined || folder === "") { + searchParams.delete(DAG_FOLDER); + } else { + searchParams.set(DAG_FOLDER, folder); + } + if (bundleName === undefined || bundleName === "") { + searchParams.delete(DAG_BUNDLE); + } else { + searchParams.set(DAG_BUNDLE, bundleName); + } + searchParams.delete(OFFSET); + setSearchParams(searchParams); + }; + let paused = defaultShowPaused; let isFavorite = undefined; let pendingHitl = undefined; @@ -309,6 +348,7 @@ export const DagsList = () => { const { data, error, isLoading } = useDags({ advancedSearch: advancedSearch.enabled, + bundleName: selectedBundle, dagDisplayNamePattern: Boolean(dagDisplayNamePattern) ? dagDisplayNamePattern : undefined, dagRunsLimit, dagRunState, @@ -320,6 +360,7 @@ export const DagsList = () => { owners, paused, pendingHitl, + relativeFilelocPrefix: selectedFolder, tags: selectedTags, tagsMatchMode: selectedMatchMode, teams: teams.length > 0 ? teams : undefined, @@ -376,24 +417,39 @@ export const DagsList = () => { - - } - initialState={tableURLState} - isLoading={isLoading} - modelName="common:dag" - onDisplayToggleChange={setDisplay} - onStateChange={setTableURLState} - showDisplayToggle - showRowCountHeading={false} - skeletonCount={display === "card" ? 5 : undefined} - total={totalEntries} - /> - + + {/* Only show the folder sidebar when there is something to navigate; deployments with all + Dags at the bundle root would otherwise get an empty panel. */} + {showFolderTree ? ( + + + + ) : undefined} + + } + initialState={tableURLState} + isLoading={isLoading} + modelName="common:dag" + onDisplayToggleChange={setDisplay} + onStateChange={setTableURLState} + showDisplayToggle + showRowCountHeading={false} + skeletonCount={display === "card" ? 5 : undefined} + total={totalEntries} + /> + + ); }; diff --git a/airflow-core/src/airflow/ui/src/queries/useDagFolders.ts b/airflow-core/src/airflow/ui/src/queries/useDagFolders.ts new file mode 100644 index 0000000000000..918668363906f --- /dev/null +++ b/airflow-core/src/airflow/ui/src/queries/useDagFolders.ts @@ -0,0 +1,35 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { useDagServiceGetDagFolders } from "openapi/queries"; + +/** + * Fetch the distinct folders of all readable Dags, each paired with its bundle. + * + * The list powers the folder navigation tree on the Dags page; the tree hierarchy is + * reconstructed client-side by splitting each path on ``/`` and grouping by bundle. + */ +export const useDagFolders = () => { + const { data, error, isLoading } = useDagServiceGetDagFolders(); + + return { + error, + folders: data?.folders ?? [], + isLoading, + }; +}; diff --git a/airflow-core/src/airflow/ui/src/queries/useDags.tsx b/airflow-core/src/airflow/ui/src/queries/useDags.tsx index 43d2cab576969..ca6e7c6ae288e 100644 --- a/airflow-core/src/airflow/ui/src/queries/useDags.tsx +++ b/airflow-core/src/airflow/ui/src/queries/useDags.tsx @@ -22,6 +22,7 @@ import { isStatePending, useAutoRefresh } from "src/utils"; export const useDags = ({ advancedSearch = false, + bundleName, dagDisplayNamePattern, dagIdPattern, dagRunsLimit, @@ -35,11 +36,13 @@ export const useDags = ({ owners, paused, pendingHitl, + relativeFilelocPrefix, tags, tagsMatchMode, teams, }: { advancedSearch?: boolean; + bundleName?: string; dagDisplayNamePattern?: string; dagIdPattern?: string; dagRunsLimit: number; @@ -53,6 +56,7 @@ export const useDags = ({ owners?: Array; paused?: boolean; pendingHitl?: boolean; + relativeFilelocPrefix?: string; tags?: Array; tagsMatchMode?: "all" | "any"; teams?: Array; @@ -64,6 +68,7 @@ export const useDags = ({ ...(advancedSearch ? { dagDisplayNamePattern, dagIdPattern } : { dagDisplayNamePrefixPattern: dagDisplayNamePattern, dagIdPrefixPattern: dagIdPattern }), + bundleName, dagRunsLimit, dagRunState, excludeStale, @@ -75,6 +80,7 @@ export const useDags = ({ orderBy, owners, paused, + relativeFilelocPrefix, tags, tagsMatchMode, teams, diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_dags.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_dags.py index 1cd4a70de7dc9..870cdabf3f383 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_dags.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_dags.py @@ -31,6 +31,7 @@ from airflow.models import DagRun from airflow.models.dag import DagModel, DagTag from airflow.models.dag_favorite import DagFavorite +from airflow.models.dagbundle import DagBundleModel from airflow.models.hitl import HITLDetail from airflow.sdk.timezone import utcnow from airflow.utils.session import NEW_SESSION, provide_session @@ -628,3 +629,161 @@ def test_should_response_401(self, unauthenticated_test_client): def test_should_response_403(self, unauthorized_test_client): response = unauthorized_test_client.get("/dags/run_state_counts", params={"dag_ids": [DAG1_ID]}) assert response.status_code == 403 + + +# Maps dag_id -> (bundle_name, relative_fileloc). ``team_alpha`` proves a folder +# name is never matched as a prefix of another (``team_a`` must not catch it), +# ``root_dag.py`` lives at the bundle root (no folder), and ``other_bundle`` reuses +# the ``team_a/etl`` path to prove folders are kept separate per bundle. +OTHER_BUNDLE = "other_bundle" +FOLDER_DAGS = { + "folder_dag_a_etl_extract": ("dag_maker", "team_a/etl/extract.py"), + "folder_dag_a_etl_load": ("dag_maker", "team_a/etl/load.py"), + "folder_dag_a_report": ("dag_maker", "team_a/report.py"), + "folder_dag_b_ml_train": ("dag_maker", "team_b/ml/train.py"), + "folder_dag_alpha": ("dag_maker", "team_alpha/x.py"), + "folder_dag_root": ("dag_maker", "root_dag.py"), + "folder_dag_other_etl": (OTHER_BUNDLE, "team_a/etl/other.py"), +} + + +class TestDagFolders(TestPublicDagEndpoint): + @pytest.fixture(autouse=True) + @provide_session + def setup_folder_dags(self, *, session: Session = NEW_SESSION) -> None: + # The extra bundle must exist before its Dags are inserted (FK on ``bundle_name``). + session.merge(DagBundleModel(name=OTHER_BUNDLE)) + session.flush() + for dag_id, (bundle_name, relative_fileloc) in FOLDER_DAGS.items(): + session.add( + DagModel( + dag_id=dag_id, + bundle_name=bundle_name, + relative_fileloc=relative_fileloc, + fileloc=f"/tmp/{relative_fileloc}", + is_stale=False, + is_paused=False, + ) + ) + session.commit() + + def test_get_dag_folders(self, test_client): + response = test_client.get("/dags/folders") + assert response.status_code == 200 + body = response.json() + # Distinct (bundle, folder) pairs of every readable Dag, sorted. Root-level + # Dags contribute no folder, and ``team_a/etl`` exists under both bundles + # yet stays as two separate entries. + assert body["folders"] == [ + {"bundle_name": "dag_maker", "folder": "team_a"}, + {"bundle_name": "dag_maker", "folder": "team_a/etl"}, + {"bundle_name": "dag_maker", "folder": "team_alpha"}, + {"bundle_name": "dag_maker", "folder": "team_b/ml"}, + {"bundle_name": OTHER_BUNDLE, "folder": "team_a/etl"}, + ] + assert body["total_entries"] == 5 + + def test_get_dag_folders_query_count_does_not_scale_with_dags(self, session, test_client): + """The folders endpoint must run a finite number of queries regardless of how many Dags exist.""" + with count_queries() as result: + response = test_client.get("/dags/folders") + assert response.status_code == 200 + baseline = sum(result.values()) + + # Add many more Dags (in both new and existing folders); the query count must not grow. + for i in range(50): + relative_fileloc = f"team_c/sub_{i}/dag_{i}.py" + session.add( + DagModel( + dag_id=f"folder_scale_dag_{i}", + bundle_name="dag_maker", + relative_fileloc=relative_fileloc, + fileloc=f"/tmp/{relative_fileloc}", + is_stale=False, + is_paused=False, + ) + ) + session.commit() + session.expire_all() + + with count_queries() as result_after: + response = test_client.get("/dags/folders") + assert response.status_code == 200 + assert sum(result_after.values()) == baseline + + def test_get_dag_folders_should_response_401(self, unauthenticated_test_client): + response = unauthenticated_test_client.get("/dags/folders") + assert response.status_code == 401 + + def test_get_dag_folders_should_response_403(self, unauthorized_test_client): + response = unauthorized_test_client.get("/dags/folders") + assert response.status_code == 403 + + @pytest.mark.parametrize( + ("prefix", "expected_dag_ids"), + [ + pytest.param( + "team_a", + { + "folder_dag_a_etl_extract", + "folder_dag_a_etl_load", + "folder_dag_a_report", + "folder_dag_other_etl", + }, + id="folder-with-subfolders", + ), + pytest.param( + "team_a/etl", + {"folder_dag_a_etl_extract", "folder_dag_a_etl_load", "folder_dag_other_etl"}, + id="nested-folder-across-bundles", + ), + pytest.param( + "team_a/etl/", + {"folder_dag_a_etl_extract", "folder_dag_a_etl_load", "folder_dag_other_etl"}, + id="trailing-slash-normalized", + ), + pytest.param("team_b", {"folder_dag_b_ml_train"}, id="intermediate-folder"), + pytest.param("team_b/ml", {"folder_dag_b_ml_train"}, id="leaf-folder"), + pytest.param("team_alpha", {"folder_dag_alpha"}, id="sibling-prefix-folder"), + pytest.param("does/not/exist", set(), id="no-match"), + ], + ) + def test_folder_filter(self, test_client, prefix, expected_dag_ids): + # The folder filter matches on path only; bundle scoping is layered on via the + # existing ``bundle_name`` filter (see test_folder_filter_scoped_by_bundle). + response = test_client.get("/dags", params={"relative_fileloc_prefix": prefix}) + assert response.status_code == 200 + returned = {dag["dag_id"] for dag in response.json()["dags"]} + # Intersect with our Dags so pre-existing setup Dags don't affect the assertion. + assert returned & set(FOLDER_DAGS) == expected_dag_ids + # ``team_a`` must never match ``team_alpha`` (and vice-versa). + if prefix == "team_a": + assert "folder_dag_alpha" not in returned + + @pytest.mark.parametrize( + ("bundle_name", "expected_dag_ids"), + [ + pytest.param( + "dag_maker", + {"folder_dag_a_etl_extract", "folder_dag_a_etl_load"}, + id="dag_maker-bundle", + ), + pytest.param(OTHER_BUNDLE, {"folder_dag_other_etl"}, id="other-bundle"), + ], + ) + def test_folder_filter_scoped_by_bundle(self, test_client, bundle_name, expected_dag_ids): + # Selecting a folder in the UI combines the folder path with its bundle, so the + # same ``team_a/etl`` path resolves to different Dags in different bundles. + response = test_client.get( + "/dags", + params={"relative_fileloc_prefix": "team_a/etl", "bundle_name": bundle_name}, + ) + assert response.status_code == 200 + returned = {dag["dag_id"] for dag in response.json()["dags"]} + assert returned & set(FOLDER_DAGS) == expected_dag_ids + + def test_no_folder_filter_returns_all_folder_dags(self, test_client): + response = test_client.get("/dags", params={"limit": 100}) + assert response.status_code == 200 + returned = {dag["dag_id"] for dag in response.json()["dags"]} + assert set(FOLDER_DAGS) <= returned