wip: data federation

.cdsrc.yaml

@@ -0,0 +1,67 @@
+features:
+  ieee754compatible: true # REVISIT: Why do we need that ??
+"[fed-srv]":
+  features:
+    with_mocks: false
+  requires:
+    sap.capire.flights.data:
+      model: "@capire/xflights-api"
+      credentials:
+        url: http://localhost:4005/hcql/data/
+"[rep-srv]":
+  features:
+    with_mocks: false
+  requires:
+    sap.capire.flights.data:
+      kind: odata
+      model: "db/rep-srv"
+      credentials:
+        url: http://localhost:4005/odata/v4/data/
+        loadOnDemand: true
+"[fed-vrt]":
+  features:
+    with_mocks: false
+  requires:
+    db: hana
+    sap.capire.flights.data:
+      kind: odata
+      credentials:
+        remote: xflights
+        schema: XFLIGHTS_XFLIGHTS
+        supportRealTimeCDC: true
+"[fed-syn]":
+  features:
+    with_mocks: false
+  requires:
+    db: hana
+    sap.capire.flights.data:
+      kind: odata
+      vcap: xflights
+      credentials:
+        schema: XFLIGHTS_XFLIGHTS
+"[fed-ds]":
+  features:
+    with_mocks: false
+  requires:
+    db: hana
+    sap.capire.flights.data:
+      kind: odata
+      vcap:
+        name: xflights-datasphere
+      credentials:
+        sql:
+          names: quoted
+"[fed-rap]":
+  features:
+    with_mocks: false
+  requires:
+    db: hana
+    rap_v2:
+      kind: odata
+      model: db/abap
+      credentials:
+        remote: sflight-rap
+        sql:
+          names: quoted
+        service: false
+        supportRealTimeCDC: false

.github/actions/hxe/Dockerfile

@@ -0,0 +1,7 @@
+FROM saplabs/hanaexpress:latest
+
+COPY ./start-hdi.sql /usr/sap/HXE/start-hdi.sql
+COPY ./setup.sh /setup
+
+# Do initial boot
+RUN /setup

.github/actions/hxe/action.yml

@@ -0,0 +1,54 @@
+name: 'Start HANA'
+description: 'Starts an local HANA Express instance for isolated testing'
+inputs:
+  GITHUB_TOKEN:
+    description: 'Derivative token for using the GitHub REST API'
+    required: true
+outputs:
+  TAG:
+    description: "The Image Tag"
+    value: ${{ steps.find-hxe.outputs.TAG }}
+  IMAGE_ID:
+    description: "The Image ID"
+    value: ${{ steps.find-hxe.outputs.IMAGE_ID }}
+runs:
+  using: "composite"
+  steps:
+    - name: Find HXE image
+      id: find-hxe
+      shell: bash
+      run: |
+        TAG="$(sha1sum ${{ github.action_path }}/* | sha1sum --tag | grep '[^ ]*$' -o)";
+        IMAGE_ID=ghcr.io/${{ github.repository_owner }}/hanaexpress;
+        IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]');
+        echo "TAG=${TAG}" >> $GITHUB_OUTPUT;
+        echo "IMAGE_ID=${IMAGE_ID}" >> $GITHUB_OUTPUT;
+        GHCR_TOKEN=$(echo ${{ inputs.GITHUB_TOKEN }} | base64);
+        if
+          curl -H "Authorization: Bearer ${GHCR_TOKEN}" https://ghcr.io/v2/${{ github.repository_owner }}/hanaexpress/manifests/$TAG | grep "MANIFEST_UNKNOWN";
+        then
+          echo "BUILD_HXE=true" >> $GITHUB_OUTPUT
+        else
+          echo "BUILD_HXE=false" >> $GITHUB_OUTPUT
+        fi;
+    - name: Set up Docker Buildx
+      if: ${{ steps.find-hxe.outputs.BUILD_HXE == 'true' }}
+      uses: docker/setup-buildx-action@v3
+    - name: Build HXE image
+      if: ${{ steps.find-hxe.outputs.BUILD_HXE == 'true' }}
+      shell: bash
+      run: |
+        echo "${{ inputs.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin;
+        DOCKER_BUILDKIT=1 docker build -t $IMAGE_ID:$TAG ${{ github.action_path }}/;
+        docker push $IMAGE_ID:$TAG;
+      env:
+        TAG: ${{ steps.find-hxe.outputs.TAG }}
+        IMAGE_ID: ${{ steps.find-hxe.outputs.IMAGE_ID }}
+    - name: Start HXE image
+      shell: bash
+      run: |
+        echo "${{ inputs.GITHUB_TOKEN }}" | docker login ghcr.io -u $ --password-stdin;
+        { ${{ github.action_path }}/start.sh; } &
+      env:
+        TAG: ${{ steps.find-hxe.outputs.TAG }}
+        IMAGE_ID: ${{ steps.find-hxe.outputs.IMAGE_ID }}

.github/actions/hxe/ci.yml

@@ -0,0 +1,28 @@
+# Use postgres/example user/password credentials
+version: '3.1'
+
+services:
+  hana:
+    image: ${IMAGE_ID}:${TAG}
+    restart: always
+    hostname: buildkitsandbox
+    ulimits:
+      nofile:
+        soft: 1048576
+        hard: 1048576
+    #sysctls:
+    #  - kernel.shmmax=1073741824
+    #  - net.ipv4.ip_local_port_range='60000 65535'
+    #  - kernel.shmmni=4096
+    #  - kernel.shmall=8388608
+    ports:
+      # Currently the only port being used is 39041
+      - '30041:39041'
+      # - '30013:39013'
+      # - '30015:39015'
+      # - '30041-30045:39041-39045'
+      # - '1128-1129:1128-1129'
+      # - '50013-50014:59013-59014'
+      # - '30030-30033:39030-39033'
+      # - '51000-51060:51000-51060'
+      # - '53075:53075'

.github/actions/hxe/setup.sh

@@ -0,0 +1,6 @@
+/run_hana --agree-to-sap-license --dont-check-system --dont-check-mount-points --master-password Manager1 &
+until /check_hana_health -n -e ready-status > /dev/null; do sleep 1; done;
+/usr/sap/HXE/HDB90/exe/hdbsql -i 90 -d SYSTEMDB -u SYSTEM -p Manager1 -I /usr/sap/HXE/start-hdi.sql
+
+kill -TERM -- -0
+wait

.github/actions/hxe/start-hdi.sql

@@ -0,0 +1,43 @@
+DO
+BEGIN
+  DECLARE dbName NVARCHAR(25) = 'HXE';
+  DECLARE diserverCount INT = 0;
+  DECLARE dpserverCount INT = 0;
+
+  -- Ensures that the HDI is enabled on the system
+  SELECT COUNT(*) INTO diserverCount FROM SYS_DATABASES.M_SERVICES WHERE SERVICE_NAME = 'diserver' AND DATABASE_NAME = :dbName AND ACTIVE_STATUS = 'YES';
+  IF diserverCount = 0 THEN 
+    EXEC 'ALTER DATABASE ' || :dbName || ' ADD ''diserver''';
+  END IF;
+
+  -- Ensure that remote sources are enabled on the system
+  SELECT COUNT(*) INTO dpserverCount FROM SYS_DATABASES.M_SERVICES WHERE SERVICE_NAME = 'dpserver' AND DATABASE_NAME = :dbName AND ACTIVE_STATUS = 'YES';
+  IF dpserverCount = 0 THEN 
+    EXEC 'ALTER DATABASE ' || :dbName || ' ADD ''dpserver''';
+  END IF;
+END;
+
+-- Grants HDI privileges to SYSTEM
+CREATE LOCAL TEMPORARY TABLE #PRIVILEGES LIKE _SYS_DI.TT_API_PRIVILEGES;
+INSERT INTO #PRIVILEGES (PRINCIPAL_NAME, PRIVILEGE_NAME, OBJECT_NAME) SELECT 'SYSTEM', PRIVILEGE_NAME, OBJECT_NAME FROM _SYS_DI.T_DEFAULT_DI_ADMIN_PRIVILEGES;
+CALL _SYS_DI.GRANT_CONTAINER_GROUP_API_PRIVILEGES('_SYS_DI', #PRIVILEGES, _SYS_DI.T_NO_PARAMETERS, ?, ?, ?);
+DROP TABLE #PRIVILEGES;
+
+-- Forces all statistics tables to use NSE
+CALL _SYS_STATISTICS.SHARED_ALTER_PAGE_LOADABLE;
+
+-- Selects all tables that are loaded and unloads them from memory
+DO
+BEGIN
+  DECLARE v_isbn VARCHAR(20) = '';
+  DECLARE CURSOR c_cursor1 (v_isbn VARCHAR(20)) FOR
+    SELECT schema_name,table_name FROM m_cs_tables WHERE loaded != 'NO';
+
+  FOR cur_row AS c_cursor1(v_isbn) DO
+    EXEC 'UNLOAD ' || :cur_row.schema_name || '.' || :cur_row.table_name || ' DELETE PERSISTENT MEMORY';
+  END FOR;
+END;
+
+-- Configure maximum memory allocation to 8192MiB as this does not translate to physical memory
+ALTER SYSTEM ALTER CONFIGURATION ('global.ini', 'system') SET ('memorymanager', 'global_allocation_limit') = '10240' WITH RECONFIGURE;
+ALTER SYSTEM ALTER CONFIGURATION ('indexserver.ini', 'DATABASE', 'HXE') SET ('smart_data_access', 'enable_loopback') = 'TRUE' WITH RECONFIGURE;

.github/actions/hxe/start.sh

@@ -0,0 +1,2 @@
+docker compose -f ci.yml up -d;
+docker exec hxe-hana-1 bash -c "until /check_hana_health -n -e ready-status > /dev/null; do sleep 1; done;"

.github/workflows/test.yml

@@ -0,0 +1,44 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    types: [opened, synchronize, reopened, auto_merge_enabled]
+
+permissions:
+  contents: read
+
+# Allow parallel jobs on `main`, so that each commit is tested. For PRs, run only the latest commit.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    name: Tests
+    permissions:
+      packages: write
+
+    strategy:
+      fail-fast: true
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: 'npm'
+      - run: npm ci
+      - id: hxe
+        uses: ./.github/actions/hxe
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      # testing
+      - run: npm test
+        env:
+          FORCE_COLOR: true
+          TAG: ${{ steps.hxe.outputs.TAG }}
+          IMAGE_ID: ${{ steps.hxe.outputs.IMAGE_ID }}

.plugins/abap/cds-plugin.js

@@ -0,0 +1,33 @@
+const cds = require('@sap/cds')
+
+if (cds.env.profiles.includes('fed-rap')) {
+  // Just a little hijack to inject the rap sflight model to xflights model transformation
+  const resolver = require('@sap/cds-compiler/lib/utils/moduleResolve.js')
+
+  const _makeModuleResolver = resolver.makeModuleResolver
+  resolver.makeModuleResolver = function () {
+    const ret = _makeModuleResolver.apply(this, arguments)
+    return {
+      resolveModule: async function (dep) {
+        if (dep.module in { '@capire/xflights': 1, '@capire/xflights-api': 1 }) {
+          return __dirname + '/index.cds'
+        }
+        return ret.resolveModule(dep)
+      }
+    }
+  }
+
+  const _makeModuleResolverSync = resolver.makeModuleResolverSync
+  resolver.makeModuleResolverSync = function (a, b, c, d) {
+    const ret = _makeModuleResolverSync.apply(this, arguments)
+    return {
+      resolveModule: function (dep) {
+        if (dep.module in { '@capire/xflights': 1, '@capire/xflights-api': 1 }) {
+          return __dirname + '/index.cds'
+        }
+        return ret.resolveModule(dep)
+      }
+    }
+  }
+
+}

.plugins/abap/compose.yml

@@ -0,0 +1,19 @@
+services:
+  abap:
+    # username: SAP* / DEVELOPER
+    # password: ABAPtr2022#01
+    image: sapse/abap-cloud-developer-trial:ABAPTRIAL_2022_SP01
+    restart: always
+    hostname: vhcala4hci
+    command:
+      - -agree-to-sap-license
+      - -skip-limits-check
+    ports:
+      - 3200:3200
+      - 3300:3300
+      - 8443:8443
+      - 30213:30213
+      - 50000:50000
+      - 50001:50001
+    # volumes:
+    #   - "./A4H_Multiple.txt:/opt/sap/ASABAP_license"

.plugins/abap/index.cds

@@ -0,0 +1,71 @@
+using { sap, sap.common.CodeList } from '@sap/cds/common';
+using { rap_v2 as my } from './srv/external/rap-v2';
+
+@odata @rest @hcql @data.product
+service sap.capire.flights.data {
+
+  // Serve Flights data with inlined connection details
+  entity Flights as projection on my.Flight {
+    key ConnectionID as ID,
+    to_Connection.{
+      AirlineID as airline_ID,
+      DepartureAirport as origin_ID,
+      DestinationAirport as destination_ID,
+      DepartureTime as departure,
+      ArrivalTime as arrival,
+      Distance as distance,
+      to_DepartureAirport as origin: redirected to Airlines
+        on $self.origin.ID = $self.origin_ID,
+      to_DestinationAirport as destination: redirected to Airlines
+        on $self.destination.ID = $self.destination_ID,
+    },
+    key FlightDate as date, // preserve the flight date as a key
+    PlaneType as aircraft,
+    Price as price,    CurrencyCode as currency_code,
+    MaximumSeats as maximum_seats,
+    OccupiedSeats as occupied_seats,
+    MaximumSeats - OccupiedSeats as free_seats: Integer,
+    to_Airline as airline: redirected to Airlines on $self.airline_ID = airline.ID,
+    to_Connection as flight,
+  };
+
+  // Serve Airlines, Airports, and Supplements data as is
+  entity Airlines as projection on my.Airline {
+    key AirlineID as ID,
+    null as icon: LargeBinary,
+    Name as name,
+    CurrencyCode as currency_code,
+  };
+  entity Airports as projection on my.Airport {
+    AirportID as ID,
+    Name as name,
+    City as city,
+    CountryCode as country_code,
+  };
+  entity Supplements as select from my.Supplement mixin {
+    type: Association to SupplementTypes on SupplementCategory = type.code;
+  } into {
+    key SupplementID as ID,
+    type,
+    SupplementCategory as type_code,
+    to_SupplementText[where LanguageCode = session_context('$user.locale')].Description as descr,
+    Price as price,
+    to_Currency as currency,
+    CurrencyCode as currency_code,
+  };
+
+  entity SupplementTypes : CodeList {
+    key code : String(2) enum {
+      Beverage = 'BV';
+      Meal = 'ML';
+      Luggage = 'LU';
+      Extra = 'EX';
+    }
+  }
+
+  // Serve data for common entities from @sap/cds/common
+  entity Currencies as projection on sap.common.Currencies;
+  entity Countries as projection on sap.common.Countries;
+  entity Languages as projection on sap.common.Languages;
+
+}

.plugins/abap/license.sh

@@ -0,0 +1,2 @@
+docker cp ./A4H_Multiple.txt abap-abap-1:/opt/sap/ASABAP_license
+docker exec -it abap-abap-1 /usr/local/bin/asabap_license_update

.plugins/abap/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "@capire/xflights-abap-api",
+  "version": "1.0.0"
+}