Fix pip show parsing

src/adcm_client/packer/types.py

@@ -34,6 +34,39 @@ def __init__(self, message, errors=None):
         self.errors = errors
 
 
+def _sanitize_pip_show_output(pip_show_output: str) -> "str":
+    """Returns sanitized output of pip show -f
+     pip show returns not valid yaml, e.g. for pip show ipython
+     we can see "Summary: IPython: Productive Interactive Computing" line
+     so we should extract only required fields (Location, Files and children of Files)
+    :param pip_show_output: pip show -f command output
+    :type pip_show_output: str
+    :return: only Location, Files and children of Files lines for every module
+    :rtype: str
+    """
+    # pip show returns not valid yaml, e.g. for pip show ipython
+    # we can see "Summary: IPython: Productive Interactive Computing"
+    # so we should extract only required fields (Location, Files and children of Files)
+    resulting_lines = []
+    # indicates if we try to parse children of Files section
+    in_files_section = False
+    for line in pip_show_output.splitlines():
+        # yaml doc separator or Location
+        if line == "---" or line.startswith("Location"):
+            resulting_lines.append(line)
+            in_files_section = False
+        # we want to pass Files into results and its children
+        elif line.startswith("Files"):
+            resulting_lines.append(line)
+            in_files_section = True
+        # we are inside children of Files
+        elif in_files_section and line.startswith("  "):
+            resulting_lines.append(line)
+        else:
+            in_files_section = False
+    return "\n".join(resulting_lines)
+
+
 def _get_top_dirs(image: Image, prepared_image: Image, client: DockerClient) -> "list":
     """Returns a list of paths to all top level folders
      and files of python module that must be installed in image
@@ -57,11 +90,13 @@ def _get_top_dirs(image: Image, prepared_image: Image, client: DockerClient) ->
         map(lambda x: x.split('==')[0], set(modified_module_list).difference(default_module_list))
     )
 
-    modules_data = yaml.safe_load_all(
-        client.containers.run(
-            prepared_image, f'pip show -f {" ".join(modules)}', remove=True
-        ).decode("utf-8")
-    )
+    pip_show_output = client.containers.run(
+        prepared_image, f'pip show -f {" ".join(modules)}', remove=True
+    ).decode("utf-8")
+
+    sanitized_pip_show_output = _sanitize_pip_show_output(pip_show_output)
+
+    modules_data = yaml.safe_load_all(sanitized_pip_show_output)
     return list(
         chain.from_iterable(
             map(

tests/test_smoke.py

@@ -14,6 +14,7 @@
 import inspect
 import os
 import tarfile
+import tempfile
 from pathlib import Path
 
 import allure
@@ -411,3 +412,23 @@ def test_self_password_change(sdk_client_fs: ADCMClient, adcm_api_credentials: d
         )
         user = custom_adcm_client.user(username=adcm_api_credentials["user"])
         assert user.username == adcm_api_credentials["user"]
+
+
+def test_bundle_build_with_requirements(sdk_client_fs: ADCMClient):
+    """
+    Test how we can install requirements for a bundle
+    inside cluster action we try to import them
+    """
+    with allure.step("Create cluster"):
+        with tempfile.TemporaryDirectory(dir=get_data_dir(__file__)) as tmp_dir_name:
+            bundle = sdk_client_fs.upload_from_fs(
+                get_data_dir(__file__) + "/build_with_requirements", workspace=tmp_dir_name
+            )
+        cluster = bundle.cluster_create(name="sample cluster")
+    with allure.step("Run cluster action: install"):
+        install = cluster.action(name="install")
+    with allure.step("Check action"):
+        assert install.name == "install"
+        _assert_attrs(install)
+        job = install.run()
+        assert job.wait() == "success"

tests/test_smoke_data/build_with_requirements/config.yaml

@@ -0,0 +1,29 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+- type: cluster
+  name: cluster_type_1
+  version: 1.4
+  description: "That is description"
+
+  actions:
+    install:
+      type: job
+      script_type: ansible
+      script: do.yaml
+      states:
+        available: any
+
+- type: service
+  name: service_type_1
+  version: 1.0

tests/test_smoke_data/build_with_requirements/do.yaml

@@ -0,0 +1,29 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Check imported dependency
+  hosts: localhost
+  connection: local
+  gather_facts: no
+
+  tasks:
+  - name: Import dependency and print path
+    command: /usr/bin/python
+    args:
+      stdin: |
+        import IPython
+        print(IPython.__file__)
+    register: results
+
+  - name: Show result
+    debug:
+      msg: "{{ results.stdout}}"

tests/test_smoke_data/build_with_requirements/requirements.yaml

@@ -0,0 +1,2 @@
+python_mod:
+  - ipython==8.18.1 # it has unexpected (non-yaml) pip show output (Summary: IPython: Productive Interactive Computing)

tests/test_smoke_data/build_with_requirements/spec.yaml

@@ -0,0 +1,10 @@
+---
+version: "1.0"
+editions:
+  - name: community
+    exclude:
+      - "spec.yaml"
+      - "requirements.txt"
+    preprocessors:
+      - type: python_mod_req
+        requirements: requirements.yaml