diff --git a/sdk/python/feast/cli/ui.py b/sdk/python/feast/cli/ui.py index 9fd7b24b7cd..bcac7cf2c3c 100644 --- a/sdk/python/feast/cli/ui.py +++ b/sdk/python/feast/cli/ui.py @@ -2,6 +2,8 @@ from feast.repo_operations import create_feature_store, registry_dump +VALID_MODES = ("proto", "rest", "rest-external") + @click.command() @click.option( @@ -52,6 +54,25 @@ show_default=False, help="path to TLS(SSL) certificate public key. You need to pass --key arg as well to start server in TLS mode", ) +@click.option( + "--mode", + "-m", + type=click.Choice(VALID_MODES, case_sensitive=False), + default="proto", + show_default=True, + help=( + "Data serving mode for the UI. " + "'proto' serves the registry as a protobuf blob (current default). " + "'rest' mounts the REST registry API alongside the UI. " + "'rest-external' proxies to an external REST registry API." + ), +) +@click.option( + "--rest-api-url", + type=click.STRING, + default="", + help="Base URL of an external REST registry API (required when --mode=rest-external). Example: http://registry-host:6570/api/v1", +) @click.pass_context def ui( ctx: click.Context, @@ -61,6 +82,8 @@ def ui( root_path: str = "", tls_key_path: str = "", tls_cert_path: str = "", + mode: str = "proto", + rest_api_url: str = "", ): """ Shows the Feast UI over the current directory @@ -69,8 +92,11 @@ def ui( raise click.BadParameter( "Please configure --key and --cert args to start the feature server in SSL mode." ) + if mode == "rest-external" and not rest_api_url: + raise click.BadParameter( + "--rest-api-url is required when using --mode=rest-external." + ) store = create_feature_store(ctx) - # Pass in the registry_dump method to get around a circular dependency store.serve_ui( host=host, port=port, @@ -79,4 +105,6 @@ def ui( root_path=root_path, tls_key_path=tls_key_path, tls_cert_path=tls_cert_path, + mode=mode, + rest_api_url=rest_api_url, ) diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index f95bbf10c03..dad5835d68c 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -3153,8 +3153,15 @@ def serve_ui( root_path: str = "", tls_key_path: str = "", tls_cert_path: str = "", + mode: str = "proto", + rest_api_url: str = "", ) -> None: - """Start the UI server locally""" + """Start the UI server locally + + Args: + mode: Data serving mode - 'proto' (default), 'rest', or 'rest-external'. + rest_api_url: Base URL for external REST API (required for 'rest-external' mode). + """ if flags_helper.is_test(): warnings.warn( "The Feast UI is an experimental feature. " @@ -3171,6 +3178,8 @@ def serve_ui( root_path=root_path, tls_key_path=tls_key_path, tls_cert_path=tls_cert_path, + mode=mode, + rest_api_url=rest_api_url, ) def serve_registry( diff --git a/sdk/python/feast/ui_server.py b/sdk/python/feast/ui_server.py index 99a4abc9c81..c6b89ada28d 100644 --- a/sdk/python/feast/ui_server.py +++ b/sdk/python/feast/ui_server.py @@ -1,33 +1,72 @@ import json +import logging import threading from importlib import resources as importlib_resources from typing import Callable, Optional import uvicorn -from fastapi import FastAPI, Response, status +from fastapi import FastAPI, Request, Response, status from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles import feast +logger = logging.getLogger(__name__) -def get_app( + +def _build_projects_list( store: "feast.FeatureStore", project_id: str, - registry_ttl_secs: int, - root_path: str = "", + root_path: str, + mode: str, ): - app = FastAPI() + """Build the projects list for the UI, with mode-aware registry paths.""" + discovered_projects = [] + registry = store.registry.proto() - app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], - ) + if mode == "proto": + registry_path_template = f"{root_path}/registry" + else: + registry_path_template = f"{root_path}/api/v1" + + if registry and registry.projects and len(registry.projects) > 0: + for proj in registry.projects: + if proj.spec and proj.spec.name: + discovered_projects.append( + { + "name": proj.spec.name.replace("_", " ").title(), + "description": proj.spec.description + or f"Project: {proj.spec.name}", + "id": proj.spec.name, + "registryPath": registry_path_template, + } + ) + else: + discovered_projects.append( + { + "name": "Project", + "description": "Test project", + "id": project_id, + "registryPath": registry_path_template, + } + ) + + if len(discovered_projects) > 1: + all_projects_entry = { + "name": "All Projects", + "description": "View data across all projects", + "id": "all", + "registryPath": registry_path_template, + } + discovered_projects.insert(0, all_projects_entry) + + return {"projects": discovered_projects, "mode": mode} - # Asynchronously refresh registry, notifying shutdown and canceling the active timer if the app is shutting down + +def _setup_proto_mode( + app: FastAPI, store: "feast.FeatureStore", registry_ttl_secs: int +): + """Set up the legacy proto-blob serving mode (GET /registry).""" registry_proto = None shutting_down = False active_timer: Optional[threading.Timer] = None @@ -51,57 +90,155 @@ def shutdown_event(): async_refresh() - ui_dir_ref = importlib_resources.files(__spec__.parent) / "ui/build/" # type: ignore[name-defined, arg-type] - with importlib_resources.as_file(ui_dir_ref) as ui_dir: - # Initialize with the projects-list.json file - with ui_dir.joinpath("projects-list.json").open(mode="w") as f: - # Get all projects from the registry - discovered_projects = [] - registry = store.registry.proto() - - # Use the projects list from the registry - if registry and registry.projects and len(registry.projects) > 0: - for proj in registry.projects: - if proj.spec and proj.spec.name: - discovered_projects.append( - { - "name": proj.spec.name.replace("_", " ").title(), - "description": proj.spec.description - or f"Project: {proj.spec.name}", - "id": proj.spec.name, - "registryPath": f"{root_path}/registry", - } - ) - else: - # If no projects in registry, use the current project from feature_store.yaml - discovered_projects.append( - { - "name": "Project", - "description": "Test project", - "id": project_id, - "registryPath": f"{root_path}/registry", - } - ) + @app.get("/registry") + def read_registry(): + if registry_proto is None: + return Response(status_code=status.HTTP_503_SERVICE_UNAVAILABLE) + return Response( + content=registry_proto.SerializeToString(), + media_type="application/octet-stream", + ) - # Add "All Projects" option at the beginning if there are multiple projects - if len(discovered_projects) > 1: - all_projects_entry = { - "name": "All Projects", - "description": "View data across all projects", - "id": "all", - "registryPath": f"{root_path}/registry", - } - discovered_projects.insert(0, all_projects_entry) - - projects_dict = {"projects": discovered_projects} - f.write(json.dumps(projects_dict)) + @app.get("/health") + def health(): + return ( + Response(status_code=status.HTTP_200_OK) + if registry_proto + else Response(status_code=status.HTTP_503_SERVICE_UNAVAILABLE) + ) + + +def _setup_rest_mode(app: FastAPI, store: "feast.FeatureStore", registry_ttl_secs: int): + """Mount the REST registry API routes on the UI server under /api/v1.""" + from feast.api.registry.rest import register_all_routes + from feast.registry_server import RegistryServer + + registry_proto = None + shutting_down = False + active_timer: Optional[threading.Timer] = None + + def async_refresh(): + store.refresh_registry() + nonlocal registry_proto + registry_proto = store.registry.proto() + if shutting_down: + return + nonlocal active_timer + active_timer = threading.Timer(registry_ttl_secs, async_refresh) + active_timer.start() + + @app.on_event("shutdown") + def shutdown_event(): + nonlocal shutting_down + shutting_down = True + if active_timer: + active_timer.cancel() + + async_refresh() + + grpc_handler = RegistryServer(store.registry) + + rest_app = FastAPI(root_path="/api/v1") + register_all_routes(rest_app, grpc_handler) + app.mount("/api/v1", rest_app) @app.get("/registry") def read_registry(): if registry_proto is None: + return Response(status_code=status.HTTP_503_SERVICE_UNAVAILABLE) + return Response( + content=registry_proto.SerializeToString(), + media_type="application/octet-stream", + ) + + @app.get("/health") + def health(): + return ( + Response(status_code=status.HTTP_200_OK) + if registry_proto + else Response(status_code=status.HTTP_503_SERVICE_UNAVAILABLE) + ) + + logger.info("REST registry API mounted at /api/v1") + + +def _setup_rest_external_mode( + app: FastAPI, + store: "feast.FeatureStore", + rest_api_url: str, + registry_ttl_secs: int, +): + """Reverse-proxy REST API calls to an external registry server.""" + import httpx + + rest_api_url = rest_api_url.rstrip("/") + client = httpx.AsyncClient(timeout=60.0) + + registry_proto = None + shutting_down = False + active_timer: Optional[threading.Timer] = None + + def async_refresh(): + store.refresh_registry() + nonlocal registry_proto + registry_proto = store.registry.proto() + if shutting_down: + return + nonlocal active_timer + active_timer = threading.Timer(registry_ttl_secs, async_refresh) + active_timer.start() + + @app.on_event("shutdown") + async def shutdown_event(): + nonlocal shutting_down + shutting_down = True + if active_timer: + active_timer.cancel() + await client.aclose() + + async_refresh() + + @app.api_route("/api/v1/{path:path}", methods=["GET", "POST", "PUT", "DELETE"]) + async def proxy_to_external(request: Request, path: str): + target_url = f"{rest_api_url}/{path}" + query_string = str(request.url.query) + if query_string: + target_url = f"{target_url}?{query_string}" + + headers = { + k: v + for k, v in request.headers.items() + if k.lower() not in ("host", "content-length", "transfer-encoding") + } + + body = await request.body() + + try: + resp = await client.request( + method=request.method, + url=target_url, + headers=headers, + content=body if body else None, + ) + return Response( + content=resp.content, + status_code=resp.status_code, + media_type=resp.headers.get("content-type", "application/json"), + ) + except httpx.RequestError as e: + logger.error(f"Error proxying to {target_url}: {e}") return Response( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE - ) # Service Unavailable + content=json.dumps( + {"detail": "Failed to reach the upstream registry API"} + ), + status_code=status.HTTP_502_BAD_GATEWAY, + media_type="application/json", + ) + + @app.get("/registry") + def read_registry(): + if registry_proto is None: + return Response(status_code=status.HTTP_503_SERVICE_UNAVAILABLE) return Response( content=registry_proto.SerializeToString(), media_type="application/octet-stream", @@ -115,14 +252,45 @@ def health(): else Response(status_code=status.HTTP_503_SERVICE_UNAVAILABLE) ) - # For all other paths (such as paths that would otherwise be handled by react router), pass to React + logger.info(f"REST external proxy configured → {rest_api_url}") + + +def get_app( + store: "feast.FeatureStore", + project_id: str, + registry_ttl_secs: int, + root_path: str = "", + mode: str = "proto", + rest_api_url: str = "", +): + app = FastAPI() + + app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ) + + if mode == "rest": + _setup_rest_mode(app, store, registry_ttl_secs) + elif mode == "rest-external": + _setup_rest_external_mode(app, store, rest_api_url, registry_ttl_secs) + else: + _setup_proto_mode(app, store, registry_ttl_secs) + + ui_dir_ref = importlib_resources.files(__spec__.parent) / "ui/build/" # type: ignore[name-defined, arg-type] + with importlib_resources.as_file(ui_dir_ref) as ui_dir: + projects_dict = _build_projects_list(store, project_id, root_path, mode) + with ui_dir.joinpath("projects-list.json").open(mode="w") as f: + f.write(json.dumps(projects_dict)) + @app.api_route("/p/{path_name:path}", methods=["GET"]) def catch_all(): filename = ui_dir.joinpath("index.html") - with open(filename) as f: content = f.read() - return Response(content, media_type="text/html") app.mount( @@ -144,13 +312,20 @@ def start_server( root_path: str = "", tls_key_path: str = "", tls_cert_path: str = "", + mode: str = "proto", + rest_api_url: str = "", ): app = get_app( store, project_id, registry_ttl_sec, root_path, + mode=mode, + rest_api_url=rest_api_url, ) + + logger.info(f"Starting Feast UI server in '{mode}' mode on {host}:{port}") + if tls_key_path and tls_cert_path: uvicorn.run( app, diff --git a/sdk/python/tests/unit/test_ui_server.py b/sdk/python/tests/unit/test_ui_server.py index 36389f7b860..c2436c777cc 100644 --- a/sdk/python/tests/unit/test_ui_server.py +++ b/sdk/python/tests/unit/test_ui_server.py @@ -207,3 +207,236 @@ def test_catch_all_route(ui_app_with_registry): # The route will fail due to the scope issue with ui_dir with pytest.raises(Exception): # Expecting NameError or FileNotFoundError client.get("/p/some/react/path") + + +# ---------- Mode-aware projects-list.json tests ---------- + + +def _read_projects_list(temp_dir): + """Read the projects-list.json written by get_app via the mock (ui_dir = temp_dir).""" + projects_file = os.path.join(temp_dir, "projects-list.json") + with open(projects_file) as f: + return json.load(f) + + +def test_projects_list_proto_mode(mock_feature_store): + """projects-list.json uses /registry paths and mode='proto' by default.""" + mock_registry = MagicMock() + mock_proto = MagicMock() + mock_proto.SerializeToString.return_value = b"data" + mock_proto.projects = [] + mock_registry.proto.return_value = mock_proto + mock_feature_store.registry = mock_registry + + with tempfile.TemporaryDirectory() as temp_dir: + _create_mock_ui_files(temp_dir) + + with _setup_importlib_mocks(temp_dir): + get_app(mock_feature_store, TEST_PROJECT_NAME, REGISTRY_TTL_SECS) + + data = _read_projects_list(temp_dir) + assertpy.assert_that(data["mode"]).is_equal_to("proto") + assertpy.assert_that(data["projects"][0]["registryPath"]).is_equal_to( + "/registry" + ) + + +def test_projects_list_rest_mode(mock_feature_store): + """projects-list.json uses /api/v1 paths and mode='rest' when REST mode is set.""" + mock_registry = MagicMock() + mock_proto = MagicMock() + mock_proto.SerializeToString.return_value = b"data" + mock_proto.projects = [] + mock_registry.proto.return_value = mock_proto + mock_feature_store.registry = mock_registry + + with tempfile.TemporaryDirectory() as temp_dir: + _create_mock_ui_files(temp_dir) + + with _setup_importlib_mocks(temp_dir): + get_app( + mock_feature_store, + TEST_PROJECT_NAME, + REGISTRY_TTL_SECS, + mode="rest", + ) + + data = _read_projects_list(temp_dir) + assertpy.assert_that(data["mode"]).is_equal_to("rest") + assertpy.assert_that(data["projects"][0]["registryPath"]).is_equal_to("/api/v1") + + +def test_projects_list_rest_mode_with_root_path(mock_feature_store): + """REST mode respects root_path prefix in registryPath.""" + mock_registry = MagicMock() + mock_proto = MagicMock() + mock_proto.SerializeToString.return_value = b"data" + mock_proto.projects = [] + mock_registry.proto.return_value = mock_proto + mock_feature_store.registry = mock_registry + + with tempfile.TemporaryDirectory() as temp_dir: + _create_mock_ui_files(temp_dir) + + with _setup_importlib_mocks(temp_dir): + get_app( + mock_feature_store, + TEST_PROJECT_NAME, + REGISTRY_TTL_SECS, + root_path="/feast", + mode="rest", + ) + + data = _read_projects_list(temp_dir) + assertpy.assert_that(data["projects"][0]["registryPath"]).is_equal_to( + "/feast/api/v1" + ) + + +# ---------- REST mode backward-compat: /registry and /health still work ---------- + + +def test_rest_mode_health_endpoint(mock_feature_store): + """Health endpoint works in REST mode.""" + mock_registry = MagicMock() + mock_proto = MagicMock() + mock_proto.SerializeToString.return_value = b"data" + mock_proto.projects = [] + mock_registry.proto.return_value = mock_proto + mock_feature_store.registry = mock_registry + + with tempfile.TemporaryDirectory() as temp_dir: + _create_mock_ui_files(temp_dir) + + with _setup_importlib_mocks(temp_dir): + app = get_app( + mock_feature_store, + TEST_PROJECT_NAME, + REGISTRY_TTL_SECS, + mode="rest", + ) + client = TestClient(app) + response = client.get("/health") + assertpy.assert_that(response.status_code).is_equal_to( + EXPECTED_SUCCESS_STATUS + ) + + +def test_rest_mode_registry_endpoint_backward_compat(mock_feature_store): + """/registry proto blob endpoint is still available in REST mode.""" + mock_registry = MagicMock() + mock_proto = MagicMock() + mock_proto.SerializeToString.return_value = b"proto_blob" + mock_proto.projects = [] + mock_registry.proto.return_value = mock_proto + mock_feature_store.registry = mock_registry + + with tempfile.TemporaryDirectory() as temp_dir: + _create_mock_ui_files(temp_dir) + + with _setup_importlib_mocks(temp_dir): + app = get_app( + mock_feature_store, + TEST_PROJECT_NAME, + REGISTRY_TTL_SECS, + mode="rest", + ) + client = TestClient(app) + response = client.get("/registry") + assertpy.assert_that(response.status_code).is_equal_to( + EXPECTED_SUCCESS_STATUS + ) + assertpy.assert_that(response.headers["content-type"]).is_equal_to( + "application/octet-stream" + ) + + +# ---------- rest-external proxy tests ---------- + + +def test_rest_external_mode_health_endpoint(mock_feature_store): + """Health endpoint works in rest-external mode.""" + mock_registry = MagicMock() + mock_proto = MagicMock() + mock_proto.SerializeToString.return_value = b"data" + mock_proto.projects = [] + mock_registry.proto.return_value = mock_proto + mock_feature_store.registry = mock_registry + + with tempfile.TemporaryDirectory() as temp_dir: + _create_mock_ui_files(temp_dir) + + with _setup_importlib_mocks(temp_dir): + app = get_app( + mock_feature_store, + TEST_PROJECT_NAME, + REGISTRY_TTL_SECS, + mode="rest-external", + rest_api_url="http://fake-registry:6570/api/v1", + ) + client = TestClient(app) + response = client.get("/health") + assertpy.assert_that(response.status_code).is_equal_to( + EXPECTED_SUCCESS_STATUS + ) + + +def test_rest_external_mode_proxy_unreachable(mock_feature_store): + """rest-external returns 502 when external API is unreachable.""" + from unittest.mock import AsyncMock + + import httpx + + mock_registry = MagicMock() + mock_proto = MagicMock() + mock_proto.SerializeToString.return_value = b"data" + mock_proto.projects = [] + mock_registry.proto.return_value = mock_proto + mock_feature_store.registry = mock_registry + + mock_httpx_client = AsyncMock() + mock_httpx_client.request.side_effect = httpx.ConnectError("Connection refused") + + with tempfile.TemporaryDirectory() as temp_dir: + _create_mock_ui_files(temp_dir) + + with ( + _setup_importlib_mocks(temp_dir), + patch("httpx.AsyncClient", return_value=mock_httpx_client), + ): + app = get_app( + mock_feature_store, + TEST_PROJECT_NAME, + REGISTRY_TTL_SECS, + mode="rest-external", + rest_api_url="http://fake-registry:6570/api/v1", + ) + client = TestClient(app) + response = client.get("/api/v1/projects") + assertpy.assert_that(response.status_code).is_equal_to(502) + + +def test_rest_external_mode_projects_list(mock_feature_store): + """projects-list.json mode is 'rest-external' with /api/v1 paths.""" + mock_registry = MagicMock() + mock_proto = MagicMock() + mock_proto.SerializeToString.return_value = b"data" + mock_proto.projects = [] + mock_registry.proto.return_value = mock_proto + mock_feature_store.registry = mock_registry + + with tempfile.TemporaryDirectory() as temp_dir: + _create_mock_ui_files(temp_dir) + + with _setup_importlib_mocks(temp_dir): + get_app( + mock_feature_store, + TEST_PROJECT_NAME, + REGISTRY_TTL_SECS, + mode="rest-external", + rest_api_url="http://fake:6570/api/v1", + ) + + data = _read_projects_list(temp_dir) + assertpy.assert_that(data["mode"]).is_equal_to("rest-external") + assertpy.assert_that(data["projects"][0]["registryPath"]).is_equal_to("/api/v1") diff --git a/ui/src/FeastUISansProviders.tsx b/ui/src/FeastUISansProviders.tsx index 9a2207e22dd..ce0b5c2ea5d 100644 --- a/ui/src/FeastUISansProviders.tsx +++ b/ui/src/FeastUISansProviders.tsx @@ -38,11 +38,15 @@ import { ProjectListContext, ProjectsListContextInterface, } from "./contexts/ProjectListContext"; +import DataModeContext from "./contexts/DataModeContext"; +import type { DataMode, DataModeConfig, FetchOptions } from "./contexts/DataModeContext"; interface FeastUIConfigs { tabsRegistry?: FeastTabsRegistryInterface; featureFlags?: FeatureFlags; projectListPromise?: Promise; + mode?: DataMode; + fetchOptions?: FetchOptions; } const defaultProjectListPromise = (basename: string) => { @@ -95,10 +99,16 @@ const FeastUISansProvidersInner = ({ }) => { const { colorMode } = useTheme(); + const dataModeConfig: DataModeConfig = { + mode: feastUIConfigs?.mode || "proto", + fetchOptions: feastUIConfigs?.fetchOptions, + }; + return ( - + - + + ); diff --git a/ui/src/components/ObjectsCountStats.tsx b/ui/src/components/ObjectsCountStats.tsx index bf1dd2dc9dd..180622b6ca1 100644 --- a/ui/src/components/ObjectsCountStats.tsx +++ b/ui/src/components/ObjectsCountStats.tsx @@ -1,4 +1,4 @@ -import React, { useContext } from "react"; +import React from "react"; import { EuiFlexGroup, EuiFlexItem, @@ -7,45 +7,64 @@ import { EuiTitle, EuiSpacer, } from "@elastic/eui"; -import useLoadRegistry from "../queries/useLoadRegistry"; import { useNavigate, useParams } from "react-router-dom"; -import RegistryPathContext from "../contexts/RegistryPathContext"; - -const useLoadObjectStats = () => { - const registryUrl = useContext(RegistryPathContext); - const query = useLoadRegistry(registryUrl); - - const data = - query.isSuccess && query.data - ? { - featureServices: query.data.objects.featureServices?.length || 0, - featureViews: query.data.mergedFVList.length, - entities: query.data.objects.entities?.length || 0, - dataSources: query.data.objects.dataSources?.length || 0, - } - : undefined; - - return { - ...query, - data, - }; -}; +import useResourceQuery, { + entityListPath, + featureViewListPath, + featureServiceListPath, + dataSourceListPath, + restFeatureViewsToMergedList, +} from "../queries/useResourceQuery"; +import type { genericFVType } from "../parsers/mergedFVTypes"; const statStyle = { cursor: "pointer" }; const ObjectsCountStats = () => { - const { isLoading, isSuccess, isError, data } = useLoadObjectStats(); const { projectName } = useParams(); - const navigate = useNavigate(); + const { data: featureServices, isSuccess: fsOk } = useResourceQuery({ + resourceType: "stats-fs", + project: projectName, + protoSelect: (d) => d.objects.featureServices, + restPath: featureServiceListPath(projectName), + restSelect: (d) => d.featureServices, + }); + + const { data: featureViews, isSuccess: fvOk } = useResourceQuery< + genericFVType[] + >({ + resourceType: "stats-fvs", + project: projectName, + protoSelect: (d) => d.mergedFVList, + restPath: featureViewListPath(projectName), + restSelect: restFeatureViewsToMergedList, + }); + + const { data: entities, isSuccess: entOk } = useResourceQuery({ + resourceType: "stats-ent", + project: projectName, + protoSelect: (d) => d.objects.entities, + restPath: entityListPath(projectName), + restSelect: (d) => d.entities, + }); + + const { data: dataSources, isSuccess: dsOk } = useResourceQuery({ + resourceType: "stats-ds", + project: projectName, + protoSelect: (d) => d.objects.dataSources, + restPath: dataSourceListPath(projectName), + restSelect: (d) => d.dataSources, + }); + + const allOk = fsOk && fvOk && entOk && dsOk; + return ( - {isLoading &&

Loading

} - {isError &&

There was an error in loading registry information.

} - {isSuccess && data && ( + {!allOk &&

Loading

} + {allOk && (

Registered in this Feast project are …

@@ -57,7 +76,7 @@ const ObjectsCountStats = () => { style={statStyle} onClick={() => navigate(`/p/${projectName}/feature-service`)} description="Feature Services→" - title={data.featureServices} + title={featureServices?.length || 0} reverse /> @@ -66,7 +85,7 @@ const ObjectsCountStats = () => { style={statStyle} description="Feature Views→" onClick={() => navigate(`/p/${projectName}/feature-view`)} - title={data.featureViews} + title={featureViews?.length || 0} reverse /> @@ -75,7 +94,7 @@ const ObjectsCountStats = () => { style={statStyle} description="Entities→" onClick={() => navigate(`/p/${projectName}/entity`)} - title={data.entities} + title={entities?.length || 0} reverse /> @@ -84,7 +103,7 @@ const ObjectsCountStats = () => { style={statStyle} description="Data Sources→" onClick={() => navigate(`/p/${projectName}/data-source`)} - title={data.dataSources} + title={dataSources?.length || 0} reverse /> diff --git a/ui/src/contexts/DataModeContext.tsx b/ui/src/contexts/DataModeContext.tsx new file mode 100644 index 00000000000..27209c58ea2 --- /dev/null +++ b/ui/src/contexts/DataModeContext.tsx @@ -0,0 +1,25 @@ +import React, { useContext } from "react"; + +type DataMode = "proto" | "rest" | "rest-external"; + +interface FetchOptions { + headers?: Record; + credentials?: RequestCredentials; +} + +interface DataModeConfig { + mode: DataMode; + fetchOptions?: FetchOptions; +} + +const defaultConfig: DataModeConfig = { + mode: "proto", +}; + +const DataModeContext = React.createContext(defaultConfig); + +const useDataMode = () => useContext(DataModeContext); + +export default DataModeContext; +export { useDataMode }; +export type { DataMode, DataModeConfig, FetchOptions }; diff --git a/ui/src/contexts/ProjectListContext.ts b/ui/src/contexts/ProjectListContext.ts index c42b22f6611..8d455498c3b 100644 --- a/ui/src/contexts/ProjectListContext.ts +++ b/ui/src/contexts/ProjectListContext.ts @@ -13,6 +13,7 @@ const ProjectEntrySchema = z.object({ const ProjectsListSchema = z.object({ default: z.string().optional(), projects: z.array(ProjectEntrySchema), + mode: z.enum(["proto", "rest", "rest-external"]).optional(), }); type ProjectsListType = z.infer; diff --git a/ui/src/hooks/useTagsAggregation.ts b/ui/src/hooks/useTagsAggregation.ts index 5d36fd54285..35cf3ffde77 100644 --- a/ui/src/hooks/useTagsAggregation.ts +++ b/ui/src/hooks/useTagsAggregation.ts @@ -1,13 +1,14 @@ -import { useContext, useMemo } from "react"; -import RegistryPathContext from "../contexts/RegistryPathContext"; -import useLoadRegistry from "../queries/useLoadRegistry"; +import { useMemo } from "react"; +import { useParams } from "react-router-dom"; import { feast } from "../protos"; +import useResourceQuery, { + featureViewListPath, + featureServiceListPath, +} from "../queries/useResourceQuery"; -// Usage of generic type parameter T -// https://stackoverflow.com/questions/53203409/how-to-tell-typescript-that-im-returning-an-array-of-arrays-of-the-input-type const buildTagCollection = ( array: T[], - recordExtractor: (unknownFCO: T) => Record | undefined, // Assumes that tags are always a Record + recordExtractor: (unknownFCO: T) => Record | undefined, ): Record> => { const tagCollection = array.reduce( (memo: Record>, fco: T) => { @@ -38,17 +39,18 @@ const buildTagCollection = ( }; const useFeatureViewTagsAggregation = () => { - const registryUrl = useContext(RegistryPathContext); - const query = useLoadRegistry(registryUrl); + const { projectName } = useParams(); + const query = useResourceQuery({ + resourceType: "tags-fvs", + project: projectName, + protoSelect: (d) => d.objects.featureViews, + restPath: featureViewListPath(projectName), + restSelect: (d) => d.featureViews, + }); const data = useMemo(() => { - return query.data && query.data.objects && query.data.objects.featureViews - ? buildTagCollection( - query.data.objects.featureViews!, - (fv) => { - return fv.spec?.tags!; - }, - ) + return query.data + ? buildTagCollection(query.data, (fv) => fv.spec?.tags) : undefined; }, [query.data]); @@ -59,19 +61,18 @@ const useFeatureViewTagsAggregation = () => { }; const useFeatureServiceTagsAggregation = () => { - const registryUrl = useContext(RegistryPathContext); - const query = useLoadRegistry(registryUrl); + const { projectName } = useParams(); + const query = useResourceQuery({ + resourceType: "tags-fss", + project: projectName, + protoSelect: (d) => d.objects.featureServices, + restPath: featureServiceListPath(projectName), + restSelect: (d) => d.featureServices, + }); const data = useMemo(() => { - return query.data && - query.data.objects && - query.data.objects.featureServices - ? buildTagCollection( - query.data.objects.featureServices, - (fs) => { - return fs.spec?.tags!; - }, - ) + return query.data + ? buildTagCollection(query.data, (fs) => fs.spec?.tags) : undefined; }, [query.data]); diff --git a/ui/src/pages/ProjectOverviewPage.tsx b/ui/src/pages/ProjectOverviewPage.tsx index 839fbcc5d89..89589248da3 100644 --- a/ui/src/pages/ProjectOverviewPage.tsx +++ b/ui/src/pages/ProjectOverviewPage.tsx @@ -1,4 +1,4 @@ -import React, { useContext } from "react"; +import React from "react"; import { EuiPageTemplate, EuiText, @@ -7,8 +7,6 @@ import { EuiTitle, EuiSpacer, EuiSkeletonText, - EuiEmptyPrompt, - EuiFieldSearch, EuiPanel, EuiStat, EuiCard, @@ -17,54 +15,82 @@ import { import { useDocumentTitle } from "../hooks/useDocumentTitle"; import ObjectsCountStats from "../components/ObjectsCountStats"; import ExplorePanel from "../components/ExplorePanel"; -import useLoadRegistry from "../queries/useLoadRegistry"; -import RegistryPathContext from "../contexts/RegistryPathContext"; -import RegistryVisualizationTab from "../components/RegistryVisualizationTab"; -import RegistrySearch from "../components/RegistrySearch"; +import useResourceQuery, { + restFeatureViewsToMergedList, +} from "../queries/useResourceQuery"; import { useParams, useNavigate } from "react-router-dom"; import { useLoadProjectsList } from "../contexts/ProjectListContext"; +import type { genericFVType } from "../parsers/mergedFVTypes"; + +const getItemProject = (item: any): string => + item?.project || item?.spec?.project || ""; // Component for "All Projects" view const AllProjectsDashboard = () => { - const registryUrl = useContext(RegistryPathContext); const navigate = useNavigate(); const { data: projectsData } = useLoadProjectsList(); - const { data: registryData } = useLoadRegistry(registryUrl); - if (!registryData) { + const { data: allFVs } = useResourceQuery({ + resourceType: "all-proj-fvs", + protoSelect: (d) => d.mergedFVList, + restPath: "/feature_views/all?limit=100&include_relationships=true", + restSelect: restFeatureViewsToMergedList, + }); + + const { data: allEntities } = useResourceQuery({ + resourceType: "all-proj-entities", + protoSelect: (d) => d.objects.entities, + restPath: "/entities/all?limit=100", + restSelect: (d) => d.entities, + }); + + const { data: allDS } = useResourceQuery({ + resourceType: "all-proj-ds", + protoSelect: (d) => d.objects.dataSources, + restPath: "/data_sources/all?limit=100", + restSelect: (d) => d.dataSources, + }); + + const { data: allFS } = useResourceQuery({ + resourceType: "all-proj-fs", + protoSelect: (d) => d.objects.featureServices, + restPath: "/feature_services/all?limit=100", + restSelect: (d) => d.featureServices, + }); + + const { data: allFeatures } = useResourceQuery({ + resourceType: "all-proj-features", + protoSelect: (d) => d.allFeatures, + restPath: "/features/all?limit=100", + restSelect: (d) => d.features, + }); + + const loaded = allFVs && allEntities && allDS && allFS && allFeatures; + + if (!loaded) { return ; } - // Calculate total counts across all projects const totalCounts = { - featureViews: registryData.objects.featureViews?.length || 0, - entities: registryData.objects.entities?.length || 0, - dataSources: registryData.objects.dataSources?.length || 0, - featureServices: registryData.objects.featureServices?.length || 0, - features: registryData.allFeatures?.length || 0, + featureViews: allFVs.length, + entities: allEntities.length, + dataSources: allDS.length, + featureServices: allFS.length, + features: allFeatures.length, }; - // Get projects from registry and count their objects const projects = projectsData?.projects.filter((p) => p.id !== "all") || []; const projectStats = projects.map((project) => { - const projectFVs = - registryData.objects.featureViews?.filter( - (fv: any) => fv?.spec?.project === project.id, - ) || []; - const projectEntities = - registryData.objects.entities?.filter( - (e: any) => e?.spec?.project === project.id, - ) || []; - const projectFeatures = - registryData.allFeatures?.filter((f: any) => f?.project === project.id) || - []; + const matchesProject = (item: any) => getItemProject(item) === project.id; return { ...project, counts: { - featureViews: projectFVs.length, - entities: projectEntities.length, - features: projectFeatures.length, + featureViews: allFVs.filter((fv) => + matchesProject(fv.object || fv), + ).length, + entities: allEntities.filter(matchesProject).length, + features: allFeatures.filter(matchesProject).length, }, }; }); @@ -195,112 +221,59 @@ const AllProjectsDashboard = () => { const ProjectOverviewPage = () => { useDocumentTitle("Feast Home"); - const registryUrl = useContext(RegistryPathContext); const { projectName } = useParams<{ projectName: string }>(); - const { isLoading, isSuccess, isError, data } = useLoadRegistry( - registryUrl, - projectName, - ); + const { data: projectsData } = useLoadProjectsList(); // Show aggregated dashboard for "All Projects" view if (projectName === "all") { return ; } - const categories = [ - { - name: "Data Sources", - data: data?.objects.dataSources || [], - getLink: (item: any) => `/p/${projectName}/data-source/${item.name}`, - }, - { - name: "Entities", - data: data?.objects.entities || [], - getLink: (item: any) => `/p/${projectName}/entity/${item.name}`, - }, - { - name: "Features", - data: data?.allFeatures || [], - getLink: (item: any) => { - const featureView = item?.featureView; - return featureView - ? `/p/${projectName}/feature-view/${featureView}/feature/${item.name}` - : "#"; - }, - }, - { - name: "Feature Views", - data: data?.mergedFVList || [], - getLink: (item: any) => `/p/${projectName}/feature-view/${item.name}`, - }, - { - name: "Feature Services", - data: data?.objects.featureServices || [], - getLink: (item: any) => { - const serviceName = item?.name || item?.spec?.name; - return serviceName - ? `/p/${projectName}/feature-service/${serviceName}` - : "#"; - }, - }, - ]; + const currentProject = projectsData?.projects.find( + (p) => p.id === projectName, + ); return (

- {isLoading && } - {isSuccess && data?.project && `Project: ${data.project}`} + {currentProject + ? `Project: ${currentProject.name}` + : projectName + ? `Project: ${projectName}` + : ""}

- {isLoading && } - {isError && ( - Error Loading Project Configs} - body={ -

- There was an error loading the Project Configurations. - Please check that feature_store.yaml file is - available and well-formed. -

- } - /> + {currentProject?.description ? ( + + 
{currentProject.description}
+ + ) : ( + +

+ Welcome to your new Feast project. In this UI, you can see + Data Sources, Entities, Features, Feature Views, and Feature + Services registered in Feast. +

+

+ It looks like this project already has some objects registered. + If you are new to this project, we suggest starting by + exploring the Feature Services, as they represent the + collection of Feature Views serving a particular model. +

+

+ Note: We encourage you to replace this + welcome message with more suitable content for your team. You + can do so by specifying a project_description in + your feature_store.yaml file. +

+ )} - {isSuccess && - (data?.description ? ( - - 
{data.description}
- - ) : ( - -

- Welcome to your new Feast project. In this UI, you can see - Data Sources, Entities, Features, Feature Views, and Feature - Services registered in Feast. -

-

- It looks like this project already has some objects - registered. If you are new to this project, we suggest - starting by exploring the Feature Services, as they - represent the collection of Feature Views serving a - particular model. -

-

- Note: We encourage you to replace this - welcome message with more suitable content for your team. - You can do so by specifying a{" "} - project_description in your{" "} - feature_store.yaml file. -

- - ))} diff --git a/ui/src/pages/Sidebar.tsx b/ui/src/pages/Sidebar.tsx index 55c8ec805c9..755c4f487bf 100644 --- a/ui/src/pages/Sidebar.tsx +++ b/ui/src/pages/Sidebar.tsx @@ -1,10 +1,17 @@ -import React, { useContext, useState } from "react"; +import React, { useState } from "react"; import { EuiIcon, EuiSideNav, htmlIdGenerator } from "@elastic/eui"; import { Link, useParams } from "react-router-dom"; import { useMatchSubpath } from "../hooks/useMatchSubpath"; -import useLoadRegistry from "../queries/useLoadRegistry"; -import RegistryPathContext from "../contexts/RegistryPathContext"; +import useResourceQuery, { + entityListPath, + featureViewListPath, + featureServiceListPath, + dataSourceListPath, + savedDatasetListPath, + featuresListPath, + restFeatureViewsToMergedList, +} from "../queries/useResourceQuery"; import { DataSourceIcon } from "../graphics/DataSourceIcon"; import { EntityIcon } from "../graphics/EntityIcon"; @@ -14,11 +21,64 @@ import { DatasetIcon } from "../graphics/DatasetIcon"; import { FeatureIcon } from "../graphics/FeatureIcon"; import { HomeIcon } from "../graphics/HomeIcon"; import { PermissionsIcon } from "../graphics/PermissionsIcon"; +import type { genericFVType } from "../parsers/mergedFVTypes"; const SideNav = () => { - const registryUrl = useContext(RegistryPathContext); const { projectName } = useParams(); - const { isSuccess, data } = useLoadRegistry(registryUrl, projectName); + + const { isSuccess: dsSuccess, data: dataSources } = useResourceQuery({ + resourceType: "sidebar-ds", + project: projectName, + protoSelect: (d) => d.objects.dataSources, + restPath: dataSourceListPath(projectName), + restSelect: (d) => d.dataSources, + }); + + const { isSuccess: entSuccess, data: entities } = useResourceQuery({ + resourceType: "sidebar-entities", + project: projectName, + protoSelect: (d) => d.objects.entities, + restPath: entityListPath(projectName), + restSelect: (d) => d.entities, + }); + + const { isSuccess: fvSuccess, data: featureViews } = useResourceQuery< + genericFVType[] + >({ + resourceType: "sidebar-fvs", + project: projectName, + protoSelect: (d) => d.mergedFVList, + restPath: featureViewListPath(projectName), + restSelect: restFeatureViewsToMergedList, + }); + + const { isSuccess: featSuccess, data: features } = useResourceQuery({ + resourceType: "sidebar-features", + project: projectName, + protoSelect: (d) => d.allFeatures, + restPath: featuresListPath(projectName), + restSelect: (d) => d.features, + }); + + const { isSuccess: fsSuccess, data: featureServices } = useResourceQuery< + any[] + >({ + resourceType: "sidebar-fs", + project: projectName, + protoSelect: (d) => d.objects.featureServices, + restPath: featureServiceListPath(projectName), + restSelect: (d) => d.featureServices, + }); + + const { isSuccess: sdSuccess, data: savedDatasets } = useResourceQuery< + any[] + >({ + resourceType: "sidebar-sd", + project: projectName, + protoSelect: (d) => d.objects.savedDatasets, + restPath: savedDatasetListPath(projectName), + restSelect: (d) => d.savedDatasets, + }); const [isSideNavOpenOnMobile, setisSideNavOpenOnMobile] = useState(false); @@ -26,41 +86,12 @@ const SideNav = () => { setisSideNavOpenOnMobile(!isSideNavOpenOnMobile); }; - const dataSourcesLabel = `Data Sources ${ - isSuccess && data?.objects.dataSources - ? `(${data?.objects.dataSources?.length})` - : "" - }`; - - const entitiesLabel = `Entities ${ - isSuccess && data?.objects.entities - ? `(${data?.objects.entities?.length})` - : "" - }`; - - const featureViewsLabel = `Feature Views ${ - isSuccess && data?.mergedFVList && data?.mergedFVList.length > 0 - ? `(${data?.mergedFVList.length})` - : "" - }`; - - const featureListLabel = `Features ${ - isSuccess && data?.allFeatures && data?.allFeatures.length > 0 - ? `(${data?.allFeatures.length})` - : "" - }`; - - const featureServicesLabel = `Feature Services ${ - isSuccess && data?.objects.featureServices - ? `(${data?.objects.featureServices?.length})` - : "" - }`; - - const savedDatasetsLabel = `Datasets ${ - isSuccess && data?.objects.savedDatasets - ? `(${data?.objects.savedDatasets?.length})` - : "" - }`; + const dataSourcesLabel = `Data Sources ${dsSuccess && dataSources ? `(${dataSources.length})` : ""}`; + const entitiesLabel = `Entities ${entSuccess && entities ? `(${entities.length})` : ""}`; + const featureViewsLabel = `Feature Views ${fvSuccess && featureViews && featureViews.length > 0 ? `(${featureViews.length})` : ""}`; + const featureListLabel = `Features ${featSuccess && features && features.length > 0 ? `(${features.length})` : ""}`; + const featureServicesLabel = `Feature Services ${fsSuccess && featureServices ? `(${featureServices.length})` : ""}`; + const savedDatasetsLabel = `Datasets ${sdSuccess && savedDatasets ? `(${savedDatasets.length})` : ""}`; const baseUrl = `/p/${projectName}`; @@ -103,7 +134,9 @@ const SideNav = () => { name: featureListLabel, id: htmlIdGenerator("featureList")(), icon: , - renderItem: (props) => , + renderItem: (props) => ( + + ), isSelected: useMatchSubpath(`${baseUrl}/features`), }, { @@ -128,7 +161,9 @@ const SideNav = () => { name: savedDatasetsLabel, id: htmlIdGenerator("savedDatasets")(), icon: , - renderItem: (props) => , + renderItem: (props) => ( + + ), isSelected: useMatchSubpath(`${baseUrl}/data-set`), }, { diff --git a/ui/src/pages/data-sources/DataSourceOverviewTab.tsx b/ui/src/pages/data-sources/DataSourceOverviewTab.tsx index d702034a558..8d570f3f26d 100644 --- a/ui/src/pages/data-sources/DataSourceOverviewTab.tsx +++ b/ui/src/pages/data-sources/DataSourceOverviewTab.tsx @@ -86,7 +86,7 @@ const DataSourceOverviewTab = () => { { + data?.requestDataOptions?.schema!.map((obj: any) => { return { fieldName: obj.name!, valueType: obj.valueType!, @@ -109,7 +109,7 @@ const DataSourceOverviewTab = () => { {consumingFeatureViews && consumingFeatureViews.length > 0 ? ( { + fvNames={consumingFeatureViews.map((f: any) => { return f.target.name; })} /> diff --git a/ui/src/pages/data-sources/Index.tsx b/ui/src/pages/data-sources/Index.tsx index 96aef712aec..821bed6e671 100644 --- a/ui/src/pages/data-sources/Index.tsx +++ b/ui/src/pages/data-sources/Index.tsx @@ -1,4 +1,4 @@ -import React, { useContext } from "react"; +import React from "react"; import { useParams } from "react-router-dom"; import { @@ -11,30 +11,26 @@ import { EuiSpacer, } from "@elastic/eui"; -import useLoadRegistry from "../../queries/useLoadRegistry"; import DatasourcesListingTable from "./DataSourcesListingTable"; import { useDocumentTitle } from "../../hooks/useDocumentTitle"; -import RegistryPathContext from "../../contexts/RegistryPathContext"; import DataSourceIndexEmptyState from "./DataSourceIndexEmptyState"; import { DataSourceIcon } from "../../graphics/DataSourceIcon"; import { useSearchQuery } from "../../hooks/useSearchInputWithTags"; import { feast } from "../../protos"; import ExportButton from "../../components/ExportButton"; +import useResourceQuery, { + dataSourceListPath, +} from "../../queries/useResourceQuery"; const useLoadDatasources = () => { - const registryUrl = useContext(RegistryPathContext); const { projectName } = useParams(); - const registryQuery = useLoadRegistry(registryUrl, projectName); - - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.objects.dataSources; - - return { - ...registryQuery, - data, - }; + return useResourceQuery({ + resourceType: "data-sources-list", + project: projectName, + protoSelect: (d) => d.objects.dataSources, + restPath: dataSourceListPath(projectName), + restSelect: (d) => d.dataSources, + }); }; const filterFn = (data: feast.core.IDataSource[], searchTokens: string[]) => { diff --git a/ui/src/pages/data-sources/useLoadDataSource.ts b/ui/src/pages/data-sources/useLoadDataSource.ts index 43f697fca03..6c7dee720c3 100644 --- a/ui/src/pages/data-sources/useLoadDataSource.ts +++ b/ui/src/pages/data-sources/useLoadDataSource.ts @@ -2,34 +2,52 @@ import { useContext } from "react"; import { useParams } from "react-router-dom"; import RegistryPathContext from "../../contexts/RegistryPathContext"; import { FEAST_FCO_TYPES } from "../../parsers/types"; -import useLoadRegistry from "../../queries/useLoadRegistry"; +import { useResolvedMode } from "../../queries/useLoadRegistry"; +import useResourceQuery, { + dataSourceDetailPath, +} from "../../queries/useResourceQuery"; const useLoadDataSource = (dataSourceName: string) => { - const registryUrl = useContext(RegistryPathContext); const { projectName } = useParams(); - const registryQuery = useLoadRegistry(registryUrl, projectName); + const mode = useResolvedMode(); - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.objects.dataSources?.find( - (ds) => ds.name === dataSourceName, - ); + const dsQuery = useResourceQuery({ + resourceType: `data-source:${dataSourceName}`, + project: projectName, + protoSelect: (d) => ({ + dataSource: d.objects.dataSources?.find( + (ds: any) => ds.name === dataSourceName, + ), + relationships: d.relationships, + }), + restPath: dataSourceDetailPath(dataSourceName, projectName || ""), + restSelect: (d) => ({ + dataSource: d, + relationships: d?.relationships || [], + }), + enabled: !!dataSourceName, + }); + + const dataSource = dsQuery.data?.dataSource; + const relationships = dsQuery.data?.relationships || []; const consumingFeatureViews = - registryQuery.data === undefined - ? undefined - : registryQuery.data.relationships.filter((relationship) => { - return ( + mode === "proto" + ? relationships.filter( + (relationship: any) => relationship.source.type === FEAST_FCO_TYPES.dataSource && - relationship.source.name === data?.name && - relationship.target.type === FEAST_FCO_TYPES.featureView - ); - }); + relationship.source.name === dataSource?.name && + relationship.target.type === FEAST_FCO_TYPES.featureView, + ) + : relationships.filter( + (rel: any) => + rel?.source?.type === "dataSource" && + rel?.target?.type === "featureView", + ); return { - ...registryQuery, - data, + ...dsQuery, + data: dataSource, consumingFeatureViews, }; }; diff --git a/ui/src/pages/entities/Index.tsx b/ui/src/pages/entities/Index.tsx index 070c53d38fa..3ca2ff09fc4 100644 --- a/ui/src/pages/entities/Index.tsx +++ b/ui/src/pages/entities/Index.tsx @@ -1,31 +1,27 @@ -import React, { useContext } from "react"; +import React from "react"; import { useParams } from "react-router-dom"; import { EuiPageTemplate, EuiLoadingSpinner } from "@elastic/eui"; import { EntityIcon } from "../../graphics/EntityIcon"; -import useLoadRegistry from "../../queries/useLoadRegistry"; import EntitiesListingTable from "./EntitiesListingTable"; import { useDocumentTitle } from "../../hooks/useDocumentTitle"; -import RegistryPathContext from "../../contexts/RegistryPathContext"; import EntityIndexEmptyState from "./EntityIndexEmptyState"; import ExportButton from "../../components/ExportButton"; +import useResourceQuery, { + entityListPath, +} from "../../queries/useResourceQuery"; const useLoadEntities = () => { - const registryUrl = useContext(RegistryPathContext); const { projectName } = useParams(); - const registryQuery = useLoadRegistry(registryUrl, projectName); - - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.objects.entities; - - return { - ...registryQuery, - data, - }; + return useResourceQuery({ + resourceType: "entities-list", + project: projectName, + protoSelect: (d) => d.objects.entities, + restPath: entityListPath(projectName), + restSelect: (d) => d.entities, + }); }; const Index = () => { diff --git a/ui/src/pages/entities/useLoadEntity.ts b/ui/src/pages/entities/useLoadEntity.ts index fdb4a7968f1..d31602c66e0 100644 --- a/ui/src/pages/entities/useLoadEntity.ts +++ b/ui/src/pages/entities/useLoadEntity.ts @@ -1,24 +1,20 @@ -import { useContext } from "react"; import { useParams } from "react-router-dom"; -import RegistryPathContext from "../../contexts/RegistryPathContext"; -import useLoadRegistry from "../../queries/useLoadRegistry"; +import useResourceQuery, { + entityDetailPath, +} from "../../queries/useResourceQuery"; const useLoadEntity = (entityName: string) => { - const registryUrl = useContext(RegistryPathContext); const { projectName } = useParams(); - const registryQuery = useLoadRegistry(registryUrl, projectName); - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.objects.entities?.find( - (fv) => fv?.spec?.name === entityName, - ); - - return { - ...registryQuery, - data, - }; + return useResourceQuery({ + resourceType: `entity:${entityName}`, + project: projectName, + protoSelect: (d) => + d.objects.entities?.find((e: any) => e?.spec?.name === entityName), + restPath: entityDetailPath(entityName, projectName || ""), + restSelect: (d) => d, + enabled: !!entityName, + }); }; export default useLoadEntity; diff --git a/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx b/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx index be922e41261..c439d48fc96 100644 --- a/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx +++ b/ui/src/pages/feature-services/FeatureServiceOverviewTab.tsx @@ -36,7 +36,7 @@ const FeatureServiceOverviewTab = () => { let numFeatures = 0; let numFeatureViews = 0; if (data) { - data?.spec?.features?.forEach((featureView) => { + data?.spec?.features?.forEach((featureView: any) => { numFeatureViews += 1; numFeatures += featureView?.featureColumns!.length; }); @@ -159,7 +159,7 @@ const FeatureServiceOverviewTab = () => { {data?.spec?.features?.length! > 0 ? ( { + data?.spec?.features?.map((f: any) => { return f.featureViewName!; })! } diff --git a/ui/src/pages/feature-services/Index.tsx b/ui/src/pages/feature-services/Index.tsx index 260a9b821dc..0aec7b91162 100644 --- a/ui/src/pages/feature-services/Index.tsx +++ b/ui/src/pages/feature-services/Index.tsx @@ -1,4 +1,4 @@ -import React, { useContext } from "react"; +import React from "react"; import { useParams } from "react-router-dom"; import { @@ -13,7 +13,6 @@ import { import { FeatureServiceIcon } from "../../graphics/FeatureServiceIcon"; -import useLoadRegistry from "../../queries/useLoadRegistry"; import FeatureServiceListingTable from "./FeatureServiceListingTable"; import { useSearchQuery, @@ -22,27 +21,24 @@ import { tagTokenGroupsType, } from "../../hooks/useSearchInputWithTags"; import { useDocumentTitle } from "../../hooks/useDocumentTitle"; -import RegistryPathContext from "../../contexts/RegistryPathContext"; import FeatureServiceIndexEmptyState from "./FeatureServiceIndexEmptyState"; import TagSearch from "../../components/TagSearch"; import ExportButton from "../../components/ExportButton"; import { useFeatureServiceTagsAggregation } from "../../hooks/useTagsAggregation"; import { feast } from "../../protos"; +import useResourceQuery, { + featureServiceListPath, +} from "../../queries/useResourceQuery"; const useLoadFeatureServices = () => { - const registryUrl = useContext(RegistryPathContext); const { projectName } = useParams(); - const registryQuery = useLoadRegistry(registryUrl, projectName); - - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.objects.featureServices; - - return { - ...registryQuery, - data, - }; + return useResourceQuery({ + resourceType: "feature-services-list", + project: projectName, + protoSelect: (d) => d.objects.featureServices, + restPath: featureServiceListPath(projectName), + restSelect: (d) => d.featureServices, + }); }; const shouldIncludeFSsGivenTokenGroups = ( diff --git a/ui/src/pages/feature-services/useLoadFeatureService.ts b/ui/src/pages/feature-services/useLoadFeatureService.ts index 004ab35b927..5574ee35077 100644 --- a/ui/src/pages/feature-services/useLoadFeatureService.ts +++ b/ui/src/pages/feature-services/useLoadFeatureService.ts @@ -1,52 +1,71 @@ import { FEAST_FCO_TYPES } from "../../parsers/types"; -import { useContext } from "react"; import { useParams } from "react-router-dom"; -import RegistryPathContext from "../../contexts/RegistryPathContext"; - -import useLoadRegistry from "../../queries/useLoadRegistry"; import { EntityReference } from "../../parsers/parseEntityRelationships"; +import { useResolvedMode } from "../../queries/useLoadRegistry"; +import useResourceQuery, { + featureServiceDetailPath, +} from "../../queries/useResourceQuery"; const useLoadFeatureService = (featureServiceName: string) => { - const registryUrl = useContext(RegistryPathContext); const { projectName } = useParams(); - const registryQuery = useLoadRegistry(registryUrl, projectName); + const mode = useResolvedMode(); - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.objects.featureServices?.find( - (fs) => fs?.spec?.name === featureServiceName, - ); + const fsQuery = useResourceQuery({ + resourceType: `feature-service:${featureServiceName}`, + project: projectName, + protoSelect: (d) => ({ + featureService: d.objects.featureServices?.find( + (fs: any) => fs?.spec?.name === featureServiceName, + ), + indirectRelationships: d.indirectRelationships, + permissions: d.permissions, + }), + restPath: featureServiceDetailPath(featureServiceName, projectName || ""), + restSelect: (d) => ({ + featureService: d, + indirectRelationships: d?.relationships || [], + permissions: d?.permissions || [], + }), + enabled: !!featureServiceName, + }); + + const featureService = fsQuery.data?.featureService; + const indirectRelationships = fsQuery.data?.indirectRelationships || []; + const permissions = fsQuery.data?.permissions || []; - let entities = - data === undefined + let entities: EntityReference[] | undefined = + featureService === undefined ? undefined - : registryQuery.data?.indirectRelationships - .filter((relationship) => { - return ( - relationship.target.type === FEAST_FCO_TYPES.featureService && - relationship.target.name === data?.spec?.name && - relationship.source.type === FEAST_FCO_TYPES.entity - ); - }) - .map((relationship) => { - return relationship.source; - }); - // Deduplicate on name of entity + : mode === "proto" + ? indirectRelationships + .filter( + (relationship: any) => + relationship.target.type === + FEAST_FCO_TYPES.featureService && + relationship.target.name === featureService?.spec?.name && + relationship.source.type === FEAST_FCO_TYPES.entity, + ) + .map((relationship: any) => relationship.source) + : indirectRelationships + .filter( + (rel: any) => + rel?.target?.type === "featureService" && + rel?.source?.type === "entity", + ) + .map((rel: any) => rel.source); + if (entities) { - let entityToName: { [key: string]: EntityReference } = {}; - for (let entity of entities) { + const entityToName: { [key: string]: EntityReference } = {}; + for (const entity of entities) { entityToName[entity.name] = entity; } entities = Object.values(entityToName); } + return { - ...registryQuery, - data: data - ? { - ...data, - permissions: registryQuery.data?.permissions, - } + ...fsQuery, + data: featureService + ? { ...featureService, permissions } : undefined, entities, }; diff --git a/ui/src/pages/feature-views/Index.tsx b/ui/src/pages/feature-views/Index.tsx index b1c28895370..418ef04fa76 100644 --- a/ui/src/pages/feature-views/Index.tsx +++ b/ui/src/pages/feature-views/Index.tsx @@ -1,4 +1,4 @@ -import React, { useContext } from "react"; +import React from "react"; import { useParams } from "react-router-dom"; import { @@ -13,7 +13,6 @@ import { import { FeatureViewIcon } from "../../graphics/FeatureViewIcon"; -import useLoadRegistry from "../../queries/useLoadRegistry"; import FeatureViewListingTable from "./FeatureViewListingTable"; import { filterInputInterface, @@ -22,26 +21,24 @@ import { } from "../../hooks/useSearchInputWithTags"; import { genericFVType, regularFVInterface } from "../../parsers/mergedFVTypes"; import { useDocumentTitle } from "../../hooks/useDocumentTitle"; -import RegistryPathContext from "../../contexts/RegistryPathContext"; import FeatureViewIndexEmptyState from "./FeatureViewIndexEmptyState"; import { useFeatureViewTagsAggregation } from "../../hooks/useTagsAggregation"; import TagSearch from "../../components/TagSearch"; import ExportButton from "../../components/ExportButton"; +import useResourceQuery, { + featureViewListPath, + restFeatureViewsToMergedList, +} from "../../queries/useResourceQuery"; const useLoadFeatureViews = () => { - const registryUrl = useContext(RegistryPathContext); const { projectName } = useParams(); - const registryQuery = useLoadRegistry(registryUrl, projectName); - - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.mergedFVList; - - return { - ...registryQuery, - data, - }; + return useResourceQuery({ + resourceType: "feature-views-list", + project: projectName, + protoSelect: (d) => d.mergedFVList, + restPath: featureViewListPath(projectName), + restSelect: restFeatureViewsToMergedList, + }); }; const shouldIncludeFVsGivenTokenGroups = ( diff --git a/ui/src/pages/feature-views/useLoadFeatureView.ts b/ui/src/pages/feature-views/useLoadFeatureView.ts index 08e8646f60f..4b5cc16c756 100644 --- a/ui/src/pages/feature-views/useLoadFeatureView.ts +++ b/ui/src/pages/feature-views/useLoadFeatureView.ts @@ -1,71 +1,69 @@ -import { useContext } from "react"; -import RegistryPathContext from "../../contexts/RegistryPathContext"; -import useLoadRegistry from "../../queries/useLoadRegistry"; +import { useParams } from "react-router-dom"; +import useResourceQuery, { + featureViewDetailPath, + restFeatureViewDetailToGeneric, +} from "../../queries/useResourceQuery"; +import type { genericFVType } from "../../parsers/mergedFVTypes"; const useLoadFeatureView = (featureViewName: string) => { - const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const { projectName } = useParams(); - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.mergedFVMap[featureViewName]; - - return { - ...registryQuery, - data, - }; + return useResourceQuery({ + resourceType: `feature-view:${featureViewName}`, + project: projectName, + protoSelect: (d) => d.mergedFVMap[featureViewName], + restPath: featureViewDetailPath(featureViewName, projectName || ""), + restSelect: restFeatureViewDetailToGeneric, + enabled: !!featureViewName, + }); }; const useLoadRegularFeatureView = (featureViewName: string) => { - const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); - - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.objects.featureViews?.find((fv) => { - return fv?.spec?.name === featureViewName; - }); + const { projectName } = useParams(); - return { - ...registryQuery, - data, - }; + return useResourceQuery({ + resourceType: `regular-fv:${featureViewName}`, + project: projectName, + protoSelect: (d) => + d.objects.featureViews?.find( + (fv: any) => fv?.spec?.name === featureViewName, + ), + restPath: featureViewDetailPath(featureViewName, projectName || ""), + restSelect: (d) => (d?.type === "featureView" ? d : undefined), + enabled: !!featureViewName, + }); }; const useLoadOnDemandFeatureView = (featureViewName: string) => { - const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const { projectName } = useParams(); - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.objects.onDemandFeatureViews?.find((fv) => { - return fv?.spec?.name === featureViewName; - }); - - return { - ...registryQuery, - data, - }; + return useResourceQuery({ + resourceType: `odfv:${featureViewName}`, + project: projectName, + protoSelect: (d) => + d.objects.onDemandFeatureViews?.find( + (fv: any) => fv?.spec?.name === featureViewName, + ), + restPath: featureViewDetailPath(featureViewName, projectName || ""), + restSelect: (d) => (d?.type === "onDemandFeatureView" ? d : undefined), + enabled: !!featureViewName, + }); }; const useLoadStreamFeatureView = (featureViewName: string) => { - const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); - - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.objects.streamFeatureViews?.find((fv) => { - return fv.spec?.name === featureViewName; - }); + const { projectName } = useParams(); - return { - ...registryQuery, - data, - }; + return useResourceQuery({ + resourceType: `sfv:${featureViewName}`, + project: projectName, + protoSelect: (d) => + d.objects.streamFeatureViews?.find( + (fv: any) => fv?.spec?.name === featureViewName, + ), + restPath: featureViewDetailPath(featureViewName, projectName || ""), + restSelect: (d) => (d?.type === "streamFeatureView" ? d : undefined), + enabled: !!featureViewName, + }); }; export default useLoadFeatureView; diff --git a/ui/src/pages/features/FeatureListPage.tsx b/ui/src/pages/features/FeatureListPage.tsx index 36087f98bc0..a9a041799fb 100644 --- a/ui/src/pages/features/FeatureListPage.tsx +++ b/ui/src/pages/features/FeatureListPage.tsx @@ -1,4 +1,4 @@ -import React, { useState, useContext } from "react"; +import React, { useState } from "react"; import { EuiBasicTable, EuiTableFieldDataColumnType, @@ -19,9 +19,10 @@ import { import EuiCustomLink from "../../components/EuiCustomLink"; import ExportButton from "../../components/ExportButton"; import { useParams } from "react-router-dom"; -import useLoadRegistry from "../../queries/useLoadRegistry"; -import RegistryPathContext from "../../contexts/RegistryPathContext"; import { FeatureIcon } from "../../graphics/FeatureIcon"; +import useResourceQuery, { + featuresListPath, +} from "../../queries/useResourceQuery"; import { FEAST_FCO_TYPES } from "../../parsers/types"; import { getEntityPermissions, @@ -43,11 +44,20 @@ type FeatureColumn = const FeatureListPage = () => { const { projectName } = useParams(); - const registryUrl = useContext(RegistryPathContext); - const { data, isLoading, isError } = useLoadRegistry( - registryUrl, - projectName, - ); + const { data: features, isLoading, isError } = useResourceQuery({ + resourceType: "features-list", + project: projectName, + protoSelect: (d) => d.allFeatures, + restPath: featuresListPath(projectName), + restSelect: (d) => d.features, + }); + const { data: permissions } = useResourceQuery({ + resourceType: "permissions", + project: projectName, + protoSelect: (d) => d.permissions, + restPath: `/permissions?project=${encodeURIComponent(projectName || "")}`, + restSelect: (d) => d.permissions, + }); const [searchText, setSearchText] = useState(""); const [selectedPermissionAction, setSelectedPermissionAction] = useState(""); @@ -57,17 +67,17 @@ const FeatureListPage = () => { const [pageIndex, setPageIndex] = useState(0); const [pageSize, setPageSize] = useState(100); - const featuresWithPermissions: Feature[] = (data?.allFeatures || []).map( + const featuresWithPermissions: Feature[] = (features || []).map( (feature) => { return { ...feature, permissions: getEntityPermissions( selectedPermissionAction ? filterPermissionsByAction( - data?.permissions, + permissions, selectedPermissionAction, ) - : data?.permissions, + : permissions, FEAST_FCO_TYPES.featureView, feature.featureView, ), @@ -75,9 +85,9 @@ const FeatureListPage = () => { }, ); - const features: Feature[] = featuresWithPermissions; + const enrichedFeatures: Feature[] = featuresWithPermissions; - const filteredFeatures = features.filter((feature) => + const filteredFeatures = enrichedFeatures.filter((feature) => feature.name.toLowerCase().includes(searchText.toLowerCase()), ); diff --git a/ui/src/pages/features/useLoadFeature.ts b/ui/src/pages/features/useLoadFeature.ts index 54bf31e996f..3f61d786152 100644 --- a/ui/src/pages/features/useLoadFeature.ts +++ b/ui/src/pages/features/useLoadFeature.ts @@ -1,27 +1,36 @@ -import { useContext } from "react"; -import RegistryPathContext from "../../contexts/RegistryPathContext"; -import useLoadRegistry from "../../queries/useLoadRegistry"; +import { useParams } from "react-router-dom"; +import useResourceQuery, { + featureDetailPath, +} from "../../queries/useResourceQuery"; const useLoadFeature = (featureViewName: string, featureName: string) => { - const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); + const { projectName } = useParams(); - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.objects.featureViews?.find((fv) => { - return fv?.spec?.name === featureViewName; - }); + const fvQuery = useResourceQuery({ + resourceType: `feature:${featureViewName}:${featureName}`, + project: projectName, + protoSelect: (d) => + d.objects.featureViews?.find( + (fv: any) => fv?.spec?.name === featureViewName, + ), + restPath: featureDetailPath( + featureViewName, + featureName, + projectName || "", + ), + restSelect: (d) => d, + enabled: !!featureViewName && !!featureName, + }); const featureData = - data === undefined + fvQuery.data === undefined ? undefined - : data?.spec?.features?.find((f) => { - return f.name === featureName; - }); + : fvQuery.data?.spec?.features?.find( + (f: any) => f.name === featureName, + ) || fvQuery.data; return { - ...registryQuery, + ...fvQuery, featureData, }; }; diff --git a/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx b/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx index 9ee7dd1aa42..d9a0ab43af9 100644 --- a/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx +++ b/ui/src/pages/saved-data-sets/DatasetOverviewTab.tsx @@ -69,7 +69,7 @@ const EntityOverviewTab = () => { { + data?.spec?.joinKeys!.map((joinKey: any) => { return { name: joinKey }; })! } diff --git a/ui/src/pages/saved-data-sets/Index.tsx b/ui/src/pages/saved-data-sets/Index.tsx index c6cc81f4146..bd161b3cfb8 100644 --- a/ui/src/pages/saved-data-sets/Index.tsx +++ b/ui/src/pages/saved-data-sets/Index.tsx @@ -1,28 +1,26 @@ -import React, { useContext } from "react"; +import React from "react"; +import { useParams } from "react-router-dom"; import { EuiPageTemplate, EuiLoadingSpinner } from "@elastic/eui"; import { DatasetIcon } from "../../graphics/DatasetIcon"; -import useLoadRegistry from "../../queries/useLoadRegistry"; import { useDocumentTitle } from "../../hooks/useDocumentTitle"; -import RegistryPathContext from "../../contexts/RegistryPathContext"; import DatasetsListingTable from "./DatasetsListingTable"; import DatasetsIndexEmptyState from "./DatasetsIndexEmptyState"; +import useResourceQuery, { + savedDatasetListPath, +} from "../../queries/useResourceQuery"; const useLoadSavedDataSets = () => { - const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); - - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.objects.savedDatasets; - - return { - ...registryQuery, - data, - }; + const { projectName } = useParams(); + return useResourceQuery({ + resourceType: "saved-datasets-list", + project: projectName, + protoSelect: (d) => d.objects.savedDatasets, + restPath: savedDatasetListPath(projectName), + restSelect: (d) => d.savedDatasets, + }); }; const Index = () => { diff --git a/ui/src/pages/saved-data-sets/useLoadDataset.ts b/ui/src/pages/saved-data-sets/useLoadDataset.ts index 40f8a8ebd48..91b90d0fbde 100644 --- a/ui/src/pages/saved-data-sets/useLoadDataset.ts +++ b/ui/src/pages/saved-data-sets/useLoadDataset.ts @@ -1,22 +1,22 @@ -import { useContext } from "react"; -import RegistryPathContext from "../../contexts/RegistryPathContext"; -import useLoadRegistry from "../../queries/useLoadRegistry"; +import { useParams } from "react-router-dom"; +import useResourceQuery, { + savedDatasetDetailPath, +} from "../../queries/useResourceQuery"; -const useLoadEntity = (entityName: string) => { - const registryUrl = useContext(RegistryPathContext); - const registryQuery = useLoadRegistry(registryUrl); +const useLoadDataset = (datasetName: string) => { + const { projectName } = useParams(); - const data = - registryQuery.data === undefined - ? undefined - : registryQuery.data.objects.savedDatasets?.find( - (fv) => fv.spec?.name === entityName, - ); - - return { - ...registryQuery, - data, - }; + return useResourceQuery({ + resourceType: `saved-dataset:${datasetName}`, + project: projectName, + protoSelect: (d) => + d.objects.savedDatasets?.find( + (sd: any) => sd.spec?.name === datasetName, + ), + restPath: savedDatasetDetailPath(datasetName, projectName || ""), + restSelect: (d) => d, + enabled: !!datasetName, + }); }; -export default useLoadEntity; +export default useLoadDataset; diff --git a/ui/src/queries/restApiClient.ts b/ui/src/queries/restApiClient.ts new file mode 100644 index 00000000000..4cf2cb64bdd --- /dev/null +++ b/ui/src/queries/restApiClient.ts @@ -0,0 +1,40 @@ +import type { FetchOptions } from "../contexts/DataModeContext"; + +class RestApiError extends Error { + status: number; + constructor(message: string, status: number) { + super(message); + this.name = "RestApiError"; + this.status = status; + } +} + +const restFetch = async ( + baseUrl: string, + path: string, + fetchOptions?: FetchOptions, +): Promise => { + const url = `${baseUrl}${path}`; + const headers: Record = { + Accept: "application/json", + ...fetchOptions?.headers, + }; + + const res = await fetch(url, { + method: "GET", + headers, + credentials: fetchOptions?.credentials, + }); + + if (!res.ok) { + throw new RestApiError( + `REST API error: ${res.status} ${res.statusText}`, + res.status, + ); + } + + return res.json(); +}; + +export default restFetch; +export { RestApiError }; diff --git a/ui/src/queries/useLoadRegistry.ts b/ui/src/queries/useLoadRegistry.ts index e3f5ac87a1d..f313e61fec5 100644 --- a/ui/src/queries/useLoadRegistry.ts +++ b/ui/src/queries/useLoadRegistry.ts @@ -5,6 +5,10 @@ import parseEntityRelationships, { } from "../parsers/parseEntityRelationships"; import parseIndirectRelationships from "../parsers/parseIndirectRelationships"; import { feast } from "../protos"; +import { useDataMode } from "../contexts/DataModeContext"; +import { useLoadProjectsList } from "../contexts/ProjectListContext"; +import restFetch from "./restApiClient"; +import type { DataMode, FetchOptions } from "../contexts/DataModeContext"; interface FeatureStoreAllData { project: string; @@ -15,7 +19,7 @@ interface FeatureStoreAllData { mergedFVList: genericFVType[]; indirectRelationships: EntityRelation[]; allFeatures: Feature[]; - permissions?: any[]; // Add permissions field + permissions?: any[]; } interface Feature { @@ -25,254 +29,384 @@ interface Feature { project?: string; } +// --------------------------------------------------------------------------- +// Shared post-processing +// --------------------------------------------------------------------------- + +const assembleFeatureStoreData = ( + objects: any, + projectName?: string, +): FeatureStoreAllData => { + const { mergedFVMap, mergedFVList } = mergedFVTypes(objects); + const relationships = parseEntityRelationships(objects); + const indirectRelationships = parseIndirectRelationships( + relationships, + objects, + ); + + const allFeatures: Feature[] = + objects.featureViews?.flatMap( + (fv: any) => + fv?.spec?.features?.map((feature: any) => ({ + name: feature.name ?? "Unknown", + featureView: fv?.spec?.name || "Unknown FeatureView", + type: + feature.valueType != null + ? typeof feature.valueType === "number" + ? feast.types.ValueType.Enum[feature.valueType] + : feature.valueType + : "Unknown Type", + project: fv?.spec?.project || fv?.project, + })) || [], + ) || []; + + let resolvedProjectName: string = + projectName === "all" + ? "All Projects" + : projectName || + (process.env.NODE_ENV === "test" + ? "credit_scoring_aws" + : objects.projects && + objects.projects.length > 0 && + objects.projects[0].spec && + objects.projects[0].spec.name + ? objects.projects[0].spec.name + : objects.project + ? objects.project + : "credit_scoring_aws"); + + let projectDescription: string | undefined; + if (projectName === "all") { + projectDescription = "View data across all projects"; + } else if (objects.projects && objects.projects.length > 0) { + const currentProject = objects.projects.find( + (p: any) => p?.spec?.name === resolvedProjectName, + ); + if (currentProject?.spec) { + projectDescription = currentProject.spec.description; + } + } + + return { + project: resolvedProjectName, + description: projectDescription, + objects, + mergedFVMap, + mergedFVList, + relationships, + indirectRelationships, + allFeatures, + permissions: + objects.permissions && objects.permissions.length > 0 + ? objects.permissions + : [ + { + spec: { + name: "zipcode-features-reader", + types: [2], + name_patterns: ["zipcode_features"], + policy: { roles: ["analyst", "data_scientist"] }, + actions: [1, 4, 5], + }, + }, + { + spec: { + name: "zipcode-source-writer", + types: [7], + name_patterns: ["zipcode"], + policy: { roles: ["admin", "data_engineer"] }, + actions: [0, 2, 7], + }, + }, + { + spec: { + name: "credit-score-v1-reader", + types: [6], + name_patterns: ["credit_score_v1"], + policy: { roles: ["model_user", "data_scientist"] }, + actions: [1, 4], + }, + }, + { + spec: { + name: "risky-features-reader", + types: [2, 6], + name_patterns: [], + required_tags: { stage: "prod" }, + policy: { roles: ["trusted_analyst"] }, + actions: [5], + }, + }, + ], + }; +}; + +// --------------------------------------------------------------------------- +// Proto fetch strategy (original behaviour) +// --------------------------------------------------------------------------- + +const fetchProto = async ( + url: string, + projectName?: string, +): Promise => { + const res = await fetch(url, { + headers: { "Content-Type": "application/json" }, + }); + + const contentType = res.headers.get("content-type"); + let data; + if (contentType && contentType.includes("application/json")) { + data = await res.json(); + } else { + data = await res.arrayBuffer(); + } + + let objects: any; + if (data instanceof ArrayBuffer) { + objects = feast.core.Registry.decode(new Uint8Array(data)); + } else { + objects = data; + } + + if (!objects.featureViews) { + objects.featureViews = []; + } + + if (projectName && projectName !== "all") { + const projectsInRegistry = new Set(); + objects.featureViews?.forEach((fv: any) => { + if (fv?.spec?.project) projectsInRegistry.add(fv.spec.project); + }); + objects.entities?.forEach((entity: any) => { + if (entity?.spec?.project) projectsInRegistry.add(entity.spec.project); + }); + + const shouldFilter = + projectsInRegistry.size > 1 || projectsInRegistry.has(projectName); + + if (shouldFilter && projectsInRegistry.has(projectName)) { + if (objects.featureViews) { + objects.featureViews = objects.featureViews.filter( + (fv: any) => fv?.spec?.project === projectName, + ); + } + if (objects.entities) { + objects.entities = objects.entities.filter( + (entity: any) => entity?.spec?.project === projectName, + ); + } + if (objects.dataSources) { + objects.dataSources = objects.dataSources.filter( + (ds: any) => ds?.project === projectName, + ); + } + if (objects.featureServices) { + objects.featureServices = objects.featureServices.filter( + (fs: any) => fs?.spec?.project === projectName, + ); + } + if (objects.onDemandFeatureViews) { + objects.onDemandFeatureViews = objects.onDemandFeatureViews.filter( + (odfv: any) => odfv?.spec?.project === projectName, + ); + } + if (objects.streamFeatureViews) { + objects.streamFeatureViews = objects.streamFeatureViews.filter( + (sfv: any) => sfv?.spec?.project === projectName, + ); + } + if (objects.savedDatasets) { + objects.savedDatasets = objects.savedDatasets.filter( + (sd: any) => sd?.spec?.project === projectName, + ); + } + if (objects.validationReferences) { + objects.validationReferences = objects.validationReferences.filter( + (vr: any) => vr?.project === projectName, + ); + } + if (objects.permissions) { + objects.permissions = objects.permissions.filter( + (perm: any) => + perm?.spec?.project === projectName || !perm?.spec?.project, + ); + } + } + } + + if ( + process.env.NODE_ENV === "test" && + objects.featureViews.length === 0 + ) { + try { + const fs = require("fs"); + const path = require("path"); + const { feast } = require("../protos"); + const registry = fs.readFileSync( + path.resolve(__dirname, "../../public/registry.db"), + ); + const parsedRegistry = feast.core.Registry.decode(registry); + if ( + parsedRegistry.featureViews && + parsedRegistry.featureViews.length > 0 + ) { + objects.featureViews = parsedRegistry.featureViews; + } + } catch (e) { + console.error("Error loading test registry:", e); + } + } + + return assembleFeatureStoreData(objects, projectName); +}; + +// --------------------------------------------------------------------------- +// REST fetch strategy (rest / rest-external) +// --------------------------------------------------------------------------- + +const fetchREST = async ( + apiBaseUrl: string, + projectName?: string, + fetchOptions?: FetchOptions, +): Promise => { + const projectParam = + projectName && projectName !== "all" + ? `?project=${encodeURIComponent(projectName)}` + : ""; + const useAllEndpoint = !projectParam; + + const [ + entitiesResp, + featureViewsResp, + featureServicesResp, + dataSourcesResp, + savedDatasetsResp, + projectsResp, + ] = await Promise.all([ + restFetch( + apiBaseUrl, + useAllEndpoint + ? "/entities/all?include_relationships=true" + : `/entities${projectParam}&include_relationships=true`, + fetchOptions, + ), + restFetch( + apiBaseUrl, + useAllEndpoint + ? "/feature_views/all?include_relationships=true" + : `/feature_views${projectParam}&include_relationships=true`, + fetchOptions, + ), + restFetch( + apiBaseUrl, + useAllEndpoint + ? "/feature_services/all?include_relationships=true" + : `/feature_services${projectParam}&include_relationships=true`, + fetchOptions, + ), + restFetch( + apiBaseUrl, + useAllEndpoint + ? "/data_sources/all?include_relationships=true" + : `/data_sources${projectParam}&include_relationships=true`, + fetchOptions, + ), + restFetch( + apiBaseUrl, + useAllEndpoint + ? "/saved_datasets/all?include_relationships=true" + : `/saved_datasets${projectParam}&include_relationships=true`, + fetchOptions, + ), + restFetch(apiBaseUrl, "/projects", fetchOptions), + ]); + + const entities = entitiesResp.entities || []; + const allFeatureViews = featureViewsResp.featureViews || []; + const featureServices = featureServicesResp.featureServices || []; + const dataSources = dataSourcesResp.dataSources || []; + const savedDatasets = savedDatasetsResp.savedDatasets || []; + const projects = projectsResp.projects || []; + + const featureViews: any[] = []; + const onDemandFeatureViews: any[] = []; + const streamFeatureViews: any[] = []; + + for (const fv of allFeatureViews) { + const fvType = fv.type; + if (fvType === "onDemandFeatureView") { + onDemandFeatureViews.push(fv); + } else if (fvType === "streamFeatureView") { + streamFeatureViews.push(fv); + } else { + featureViews.push(fv); + } + } + + const objects: any = { + entities, + featureViews, + onDemandFeatureViews, + streamFeatureViews, + featureServices, + dataSources, + savedDatasets, + projects, + }; + + return assembleFeatureStoreData(objects, projectName); +}; + +// --------------------------------------------------------------------------- +// Resolve effective mode +// --------------------------------------------------------------------------- + +const useResolvedMode = (): DataMode => { + const { mode: configMode } = useDataMode(); + const { data: projectsData } = useLoadProjectsList(); + const projectListMode = (projectsData as any)?.mode as + | DataMode + | undefined; + + if (configMode && configMode !== "proto") { + return configMode; + } + if (projectListMode) { + return projectListMode; + } + return configMode || "proto"; +}; + +// --------------------------------------------------------------------------- +// Public hook +// --------------------------------------------------------------------------- + const useLoadRegistry = (url: string, projectName?: string) => { + const resolvedMode = useResolvedMode(); + const { fetchOptions } = useDataMode(); + + // Proto mode uses the same key format as useResourceQuery so all hooks + // that need the proto registry share a single cached fetch. + const queryKey = + resolvedMode === "proto" + ? ["proto-registry", url, projectName || "all"] + : ["registry-rest-bulk", url, projectName || "all"]; + return useQuery( - `registry:${url}:${projectName || "all"}`, + queryKey, () => { - return fetch(url, { - headers: { - "Content-Type": "application/json", - }, - }) - .then((res) => { - const contentType = res.headers.get("content-type"); - if (contentType && contentType.includes("application/json")) { - return res.json(); - } else { - return res.arrayBuffer(); - } - }) - .then((data) => { - let objects; - - if (data instanceof ArrayBuffer) { - objects = feast.core.Registry.decode(new Uint8Array(data)); - } else { - objects = data; - } - // const objects = FeastRegistrySchema.parse(json); - - if (!objects.featureViews) { - objects.featureViews = []; - } - - // Filter objects by project if projectName is provided - // Skip filtering if projectName is "all" (All Projects view) - // Only filter if we detect that the registry contains multiple projects - if (projectName && projectName !== "all") { - // Check if the registry actually has multiple projects - const projectsInRegistry = new Set(); - objects.featureViews?.forEach((fv: any) => { - if (fv?.spec?.project) projectsInRegistry.add(fv.spec.project); - }); - objects.entities?.forEach((entity: any) => { - if (entity?.spec?.project) - projectsInRegistry.add(entity.spec.project); - }); - - // Only apply filtering if there are actually multiple projects in the registry - // OR if the projectName matches one of the projects in the registry - const shouldFilter = - projectsInRegistry.size > 1 || - projectsInRegistry.has(projectName); - - if (shouldFilter && projectsInRegistry.has(projectName)) { - if (objects.featureViews) { - objects.featureViews = objects.featureViews.filter( - (fv: any) => fv?.spec?.project === projectName, - ); - } - if (objects.entities) { - objects.entities = objects.entities.filter( - (entity: any) => entity?.spec?.project === projectName, - ); - } - if (objects.dataSources) { - objects.dataSources = objects.dataSources.filter( - (ds: any) => ds?.project === projectName, - ); - } - if (objects.featureServices) { - objects.featureServices = objects.featureServices.filter( - (fs: any) => fs?.spec?.project === projectName, - ); - } - if (objects.onDemandFeatureViews) { - objects.onDemandFeatureViews = - objects.onDemandFeatureViews.filter( - (odfv: any) => odfv?.spec?.project === projectName, - ); - } - if (objects.streamFeatureViews) { - objects.streamFeatureViews = objects.streamFeatureViews.filter( - (sfv: any) => sfv?.spec?.project === projectName, - ); - } - if (objects.savedDatasets) { - objects.savedDatasets = objects.savedDatasets.filter( - (sd: any) => sd?.spec?.project === projectName, - ); - } - if (objects.validationReferences) { - objects.validationReferences = - objects.validationReferences.filter( - (vr: any) => vr?.project === projectName, - ); - } - if (objects.permissions) { - objects.permissions = objects.permissions.filter( - (perm: any) => - perm?.spec?.project === projectName || !perm?.spec?.project, - ); - } - } - } - - if ( - process.env.NODE_ENV === "test" && - objects.featureViews.length === 0 - ) { - try { - const fs = require("fs"); - const path = require("path"); - const { feast } = require("../protos"); - - const registry = fs.readFileSync( - path.resolve(__dirname, "../../public/registry.db"), - ); - const parsedRegistry = feast.core.Registry.decode(registry); - - if ( - parsedRegistry.featureViews && - parsedRegistry.featureViews.length > 0 - ) { - objects.featureViews = parsedRegistry.featureViews; - } - } catch (e) { - console.error("Error loading test registry:", e); - } - } - - const { mergedFVMap, mergedFVList } = mergedFVTypes(objects); - - const relationships = parseEntityRelationships(objects); - - // Only contains Entity -> FS or DS -> FS relationships - const indirectRelationships = parseIndirectRelationships( - relationships, - objects, - ); - - // console.log({ - // objects, - // mergedFVMap, - // mergedFVList, - // relationships, - // indirectRelationships, - // }); - const allFeatures: Feature[] = - objects.featureViews?.flatMap( - (fv: any) => - fv?.spec?.features?.map((feature: any) => ({ - name: feature.name ?? "Unknown", - featureView: fv?.spec?.name || "Unknown FeatureView", - type: - feature.valueType != null - ? feast.types.ValueType.Enum[feature.valueType] - : "Unknown Type", - project: fv?.spec?.project, // Include project from parent feature view - })) || [], - ) || []; - - // Use the provided projectName parameter if available, otherwise try to determine from registry - let resolvedProjectName: string = - projectName === "all" - ? "All Projects" - : projectName || - (process.env.NODE_ENV === "test" - ? "credit_scoring_aws" - : objects.projects && - objects.projects.length > 0 && - objects.projects[0].spec && - objects.projects[0].spec.name - ? objects.projects[0].spec.name - : objects.project - ? objects.project - : "credit_scoring_aws"); - - let projectDescription = undefined; - - // Find project description from the projects array - if (projectName === "all") { - projectDescription = "View data across all projects"; - } else if (objects.projects && objects.projects.length > 0) { - const currentProject = objects.projects.find( - (p: any) => p?.spec?.name === resolvedProjectName, - ); - if (currentProject?.spec) { - projectDescription = currentProject.spec.description; - } - } - - return { - project: resolvedProjectName, - description: projectDescription, - objects, - mergedFVMap, - mergedFVList, - relationships, - indirectRelationships, - allFeatures, - permissions: - objects.permissions && objects.permissions.length > 0 - ? objects.permissions - : [ - { - spec: { - name: "zipcode-features-reader", - types: [2], // FeatureView - name_patterns: ["zipcode_features"], - policy: { roles: ["analyst", "data_scientist"] }, - actions: [1, 4, 5], // DESCRIBE, READ_ONLINE, READ_OFFLINE - }, - }, - { - spec: { - name: "zipcode-source-writer", - types: [7], // FileSource - name_patterns: ["zipcode"], - policy: { roles: ["admin", "data_engineer"] }, - actions: [0, 2, 7], // CREATE, UPDATE, WRITE_OFFLINE - }, - }, - { - spec: { - name: "credit-score-v1-reader", - types: [6], // FeatureService - name_patterns: ["credit_score_v1"], - policy: { roles: ["model_user", "data_scientist"] }, - actions: [1, 4], // DESCRIBE, READ_ONLINE - }, - }, - { - spec: { - name: "risky-features-reader", - types: [2, 6], // FeatureView, FeatureService - name_patterns: [], - required_tags: { stage: "prod" }, - policy: { roles: ["trusted_analyst"] }, - actions: [5], // READ_OFFLINE - }, - }, - ], - }; - }); + if (resolvedMode === "proto") { + return fetchProto(url, projectName); + } + return fetchREST(url, projectName, fetchOptions); }, { - staleTime: Infinity, // Given that we are reading from a registry dump, this seems reasonable for now. + staleTime: resolvedMode === "proto" ? Infinity : 30_000, + enabled: !!url, }, ); }; export default useLoadRegistry; +export { fetchProto, useResolvedMode }; export type { FeatureStoreAllData }; diff --git a/ui/src/queries/useResourceQuery.ts b/ui/src/queries/useResourceQuery.ts new file mode 100644 index 00000000000..4b2a97a20e7 --- /dev/null +++ b/ui/src/queries/useResourceQuery.ts @@ -0,0 +1,214 @@ +import { useContext } from "react"; +import { useQuery, UseQueryResult } from "react-query"; +import RegistryPathContext from "../contexts/RegistryPathContext"; +import { useDataMode } from "../contexts/DataModeContext"; +import { useResolvedMode, fetchProto } from "./useLoadRegistry"; +import restFetch from "./restApiClient"; +import type { FeatureStoreAllData } from "./useLoadRegistry"; +import { FEAST_FV_TYPES, genericFVType } from "../parsers/mergedFVTypes"; + +interface ResourceQueryOptions { + resourceType: string; + project?: string; + protoSelect: (data: FeatureStoreAllData) => T | undefined; + restPath: string; + restSelect?: (data: any) => T | undefined; + enabled?: boolean; +} + +/** + * Generic mode-aware hook for fetching a specific resource slice. + * + * Proto mode: all callers sharing the same (registryUrl, project) key + * hit one cached fetch; each caller uses `select` to extract its slice. + * + * REST mode: each caller fires its own lightweight endpoint request. + */ +function useResourceQuery({ + resourceType, + project, + protoSelect, + restPath, + restSelect, + enabled = true, +}: ResourceQueryOptions): UseQueryResult { + const mode = useResolvedMode(); + const registryUrl = useContext(RegistryPathContext); + const { fetchOptions } = useDataMode(); + + const protoResult = useQuery( + ["proto-registry", registryUrl, project || "all"], + () => fetchProto(registryUrl, project), + { + enabled: mode === "proto" && !!registryUrl && enabled, + staleTime: Infinity, + select: protoSelect, + }, + ); + + const restResult = useQuery( + ["rest", resourceType, registryUrl, project || "all"], + () => restFetch(registryUrl, restPath, fetchOptions), + { + enabled: mode !== "proto" && !!registryUrl && enabled, + staleTime: 30_000, + select: restSelect, + }, + ); + + return (mode === "proto" ? protoResult : restResult) as UseQueryResult< + T | undefined + >; +} + +// --------------------------------------------------------------------------- +// REST endpoint path builders +// --------------------------------------------------------------------------- + +function entityListPath(project?: string): string { + if (project && project !== "all") { + return `/entities?project=${encodeURIComponent(project)}&include_relationships=true`; + } + return "/entities/all?limit=100&include_relationships=true"; +} + +function entityDetailPath(name: string, project: string): string { + return `/entities/${encodeURIComponent(name)}?project=${encodeURIComponent(project)}&include_relationships=true`; +} + +function featureViewListPath(project?: string): string { + if (project && project !== "all") { + return `/feature_views?project=${encodeURIComponent(project)}&include_relationships=true`; + } + return "/feature_views/all?limit=100&include_relationships=true"; +} + +function featureViewDetailPath(name: string, project: string): string { + return `/feature_views/${encodeURIComponent(name)}?project=${encodeURIComponent(project)}&include_relationships=true`; +} + +function featureServiceListPath(project?: string): string { + if (project && project !== "all") { + return `/feature_services?project=${encodeURIComponent(project)}&include_relationships=true`; + } + return "/feature_services/all?limit=100&include_relationships=true"; +} + +function featureServiceDetailPath(name: string, project: string): string { + return `/feature_services/${encodeURIComponent(name)}?project=${encodeURIComponent(project)}&include_relationships=true`; +} + +function dataSourceListPath(project?: string): string { + if (project && project !== "all") { + return `/data_sources?project=${encodeURIComponent(project)}&include_relationships=true`; + } + return "/data_sources/all?limit=100&include_relationships=true"; +} + +function dataSourceDetailPath(name: string, project: string): string { + return `/data_sources/${encodeURIComponent(name)}?project=${encodeURIComponent(project)}&include_relationships=true`; +} + +function savedDatasetListPath(project?: string): string { + if (project && project !== "all") { + return `/saved_datasets?project=${encodeURIComponent(project)}`; + } + return "/saved_datasets/all?limit=100"; +} + +function savedDatasetDetailPath(name: string, project: string): string { + return `/saved_datasets/${encodeURIComponent(name)}?project=${encodeURIComponent(project)}`; +} + +function featuresListPath(project?: string): string { + if (project && project !== "all") { + return `/features?project=${encodeURIComponent(project)}`; + } + return "/features/all?limit=100"; +} + +function featureDetailPath( + featureViewName: string, + featureName: string, + project: string, +): string { + return `/features/${encodeURIComponent(featureViewName)}/${encodeURIComponent(featureName)}?project=${encodeURIComponent(project)}`; +} + +// --------------------------------------------------------------------------- +// REST response → mergedFVList converter +// --------------------------------------------------------------------------- + +function restFeatureViewsToMergedList(resp: any): genericFVType[] { + const featureViews = resp?.featureViews || []; + return featureViews.map((fv: any) => { + const fvType = fv.type; + if (fvType === "onDemandFeatureView") { + return { + name: fv.spec?.name, + type: FEAST_FV_TYPES.ondemand, + features: fv.spec?.features || [], + object: fv, + }; + } + if (fvType === "streamFeatureView") { + return { + name: fv.spec?.name, + type: FEAST_FV_TYPES.stream, + features: fv.spec?.features || [], + object: fv, + }; + } + return { + name: fv.spec?.name, + type: FEAST_FV_TYPES.regular, + features: fv.spec?.features || [], + object: fv, + }; + }); +} + +function restFeatureViewDetailToGeneric(resp: any): genericFVType | undefined { + if (!resp || !resp.spec) return undefined; + const fvType = resp.type; + if (fvType === "onDemandFeatureView") { + return { + name: resp.spec.name, + type: FEAST_FV_TYPES.ondemand, + features: resp.spec.features || [], + object: resp, + }; + } + if (fvType === "streamFeatureView") { + return { + name: resp.spec.name, + type: FEAST_FV_TYPES.stream, + features: resp.spec.features || [], + object: resp, + }; + } + return { + name: resp.spec.name, + type: FEAST_FV_TYPES.regular, + features: resp.spec.features || [], + object: resp, + }; +} + +export default useResourceQuery; +export { + entityListPath, + entityDetailPath, + featureViewListPath, + featureViewDetailPath, + featureServiceListPath, + featureServiceDetailPath, + dataSourceListPath, + dataSourceDetailPath, + savedDatasetListPath, + savedDatasetDetailPath, + featuresListPath, + featureDetailPath, + restFeatureViewsToMergedList, + restFeatureViewDetailToGeneric, +};