diff --git a/other/materials_designer/workflows/Introduction.ipynb b/other/materials_designer/workflows/Introduction.ipynb index 2f78cebe..eec8fc04 100644 --- a/other/materials_designer/workflows/Introduction.ipynb +++ b/other/materials_designer/workflows/Introduction.ipynb @@ -72,7 +72,7 @@ "#### 6.5.1. Defect formation energy. *(to be added)*\n", "\n", "### 6.6. Formation Energy\n", - "#### 6.6.1. Compound formation energy. *(to be added)*\n", + "#### [6.6.1. Compound formation energy.](formation_energy.ipynb)\n", "\n", "\n", "## 7. Chemistry\n", diff --git a/other/materials_designer/workflows/formation_energy.ipynb b/other/materials_designer/workflows/formation_energy.ipynb new file mode 100644 index 00000000..b77da908 --- /dev/null +++ b/other/materials_designer/workflows/formation_energy.ipynb @@ -0,0 +1,670 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Formation Energy\n", + "\n", + "Calculate the formation energy of a compound using a DFT workflow on the Mat3ra platform.\n", + "\n", + "This notebook loads a compound material, resolves Standata elemental reference materials for its constituent elements, and checks that refined `total_energy` properties already exist for each elemental reference.\n", + "\n", + "If any elemental reference material or elemental total energy is missing, the notebook stops and points to the Total Energy notebook. It does not create or run elemental pre-calculations automatically.\n", + "\n", + "

Usage

\n", + "\n", + "1. Put a compound material JSON into `../uploads`, or set `MATERIAL_NAME` to match a compound in Standata or on the platform.\n", + "2. Set the material and workflow parameters in cell 1.2 below.\n", + "3. Click \"Run\" > \"Run All\".\n", + "4. If elemental total energies are missing, run the Total Energy notebook for the corresponding Standata elemental material(s) first, then rerun this notebook.\n", + "5. Inspect the elemental references, their total energies used by the workflow, and the final formation energy.\n", + "\n", + "## Summary\n", + "\n", + "1. Set up the environment and parameters.\n", + "2. Authenticate and initialize API client.\n", + "3. Load compound material and resolve unique elements.\n", + "4. Resolve Standata elemental reference materials and check refined elemental total energies.\n", + "5. Configure and save the Formation Energy workflow.\n", + "6. Configure compute.\n", + "7. Create, submit, and monitor the Formation Energy job.\n", + "8. Retrieve results.\n" + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## 1. Set up the environment and parameters\n", + "### 1.1. Install packages (JupyterLite)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.packages import install_packages\n", + "\n", + "await install_packages(\"made|api_examples\")" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "### 1.2. Set parameters and configurations for the workflow and job\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from mat3ra.ide.compute import QueueName\n", + "\n", + "# 2. Auth and organization parameters\n", + "# Set organization name to use it as the owner, otherwise your personal account is used\n", + "ORGANIZATION_NAME = None\n", + "\n", + "# 3. Material parameters\n", + "FOLDER = \"../uploads\"\n", + "MATERIAL_NAME = \"Silicon\" # Name of the compound to load from uploads folder, Standata, or platform\n", + "\n", + "# 4. Workflow parameters\n", + "WORKFLOW_SEARCH_TERM = \"formation_energy.json\" # Search term for Workflows Standata\n", + "APPLICATION_NAME = \"espresso\" # Application for the QE formation-energy subworkflow\n", + "SCF_KGRID = None # e.g., [10, 10, 10]\n", + "MY_WORKFLOW_NAME = \"Formation Energy\"\n", + "\n", + "# 5. Compute parameters\n", + "CLUSTER_NAME = None # specify full or partial name i.e. \"cluster-001\" to select\n", + "QUEUE_NAME = QueueName.D\n", + "PPN = 1\n", + "\n", + "# 6. Job parameters\n", + "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", + "POLL_INTERVAL = 30 # seconds\n" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "## 2. Authenticate and initialize API client\n", + "### 2.1. Authenticate\n", + "Authenticate in the browser and have credentials stored in environment variable `OIDC_ACCESS_TOKEN`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.auth import authenticate\n", + "\n", + "await authenticate()" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "### 2.2. Initialize API client\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.api_client import APIClient\n", + "\n", + "client = APIClient.authenticate()\n", + "client\n" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "### 2.3. Select account to work under\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "client.list_accounts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "selected_account = client.my_account\n", + "\n", + "if ORGANIZATION_NAME:\n", + " selected_account = client.get_account(name=ORGANIZATION_NAME)\n", + "\n", + "ACCOUNT_ID = selected_account.id\n", + "print(f\"✅ Selected account ID: {ACCOUNT_ID}, name: {selected_account.name}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "12", + "metadata": {}, + "source": [ + "### 2.4. Select project\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "## 3. Load compound material\n", + "### 3.1. Load material from local file, Standata, or platform\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "from mat3ra.made.material import Material\n", + "from mat3ra.standata.materials import Materials\n", + "from mat3ra.notebooks_utils.material import load_material_from_folder\n", + "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", + "\n", + "material = load_material_from_folder(FOLDER, MATERIAL_NAME)\n", + "\n", + "if material is None:\n", + " try:\n", + " material = Material.create(Materials.get_by_name_first_match(MATERIAL_NAME))\n", + " print(f\"✅ Loaded material from Standata: {material.name}\")\n", + " except Exception:\n", + " material_matches = client.materials.list({\n", + " \"name\": {\"$regex\": re.escape(MATERIAL_NAME), \"$options\": \"i\"},\n", + " })\n", + "\n", + " material = Material.create(material_matches[0])\n", + " print(f\"♻️ Loaded material from platform: {material_matches[0]['_id']}\")\n", + "else:\n", + " print(f\"✅ Loaded material from folder: {material.name}\")\n", + "\n", + "visualize(material)" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "### 3.2. Resolve unique elements in the compound\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.ui import display_JSON\n", + "\n", + "elements = sorted(list(set(element[\"value\"] for element in material.basis.elements)))\n", + "print(f\"Compound elements: {elements}\")\n", + "display_JSON({\"elements\": elements}, level=2)" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "## 4. Resolve elemental reference materials and check total energies\n", + "### 4.1. Resolve Standata elemental reference materials\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "bank_materials_data = client.bank_materials.list(\n", + " {\"tags\": \"elemental\", \"metadata.element\": {\"$in\": elements}},\n", + " projection={\"limit\": len(elements)},\n", + ")\n", + "\n", + "element_materials = {\n", + " item[\"metadata\"][\"element\"]: item\n", + " for item in bank_materials_data\n", + " if item.get(\"metadata\") and item[\"metadata\"].get(\"element\") in elements\n", + "}\n", + "\n", + "missing_elemental_material_symbols = [element for element in elements if element not in element_materials]\n", + "if missing_elemental_material_symbols:\n", + " raise RuntimeError(\n", + " \"Missing Standata elemental reference material(s) for \"\n", + " f\"{', '.join(missing_elemental_material_symbols)}. \"\n", + " \"Import the corresponding elemental material(s) from Standata before running Formation Energy.\"\n", + " )\n", + "\n", + "print(f\"Resolved elemental reference materials for: {', '.join(elements)}\")\n", + "display_JSON({\"element_materials\": element_materials}, level=2)" + ] + }, + { + "cell_type": "markdown", + "id": "20", + "metadata": {}, + "source": [ + "### 4.2. Check refined elemental total energies\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "\n", + "def get_refined_elemental_total_energy_holders(api_client: APIClient, symbols, element_materials_by_symbol):\n", + " material_ids = [element_materials_by_symbol[symbol][\"_id\"] for symbol in symbols]\n", + " query = {\n", + " \"exabyteId\": {\"$in\": material_ids},\n", + " \"slug\": \"total_energy\",\n", + " \"systemTags\": \"isBest\",\n", + " }\n", + " results = api_client.properties.request(\n", + " \"GET\",\n", + " \"refined-properties\",\n", + " params={\n", + " \"query\": json.dumps(query),\n", + " \"projection\": json.dumps({}),\n", + " },\n", + " )\n", + " holders_by_symbol = {}\n", + " for symbol in symbols:\n", + " material_id = element_materials_by_symbol[symbol][\"_id\"]\n", + " matches = [\n", + " item\n", + " for item in results\n", + " if item.get(\"exabyteId\") == material_id or material_id in item.get(\"exabyteId\", [])\n", + " ]\n", + " holders_by_symbol[symbol] = matches[0] if matches else None\n", + " return holders_by_symbol\n", + "\n", + "\n", + "elemental_total_energy_holders = get_refined_elemental_total_energy_holders(\n", + " client,\n", + " elements,\n", + " element_materials,\n", + ")\n", + "\n", + "missing_elemental_total_energy_symbols = [\n", + " symbol for symbol in elements if elemental_total_energy_holders[symbol] is None\n", + "]\n", + "if missing_elemental_total_energy_symbols:\n", + " raise RuntimeError(\n", + " \"Missing total_energy for elemental reference material(s) \"\n", + " f\"{', '.join(missing_elemental_total_energy_symbols)}. \"\n", + " \"Run total_energy.ipynb for the corresponding Standata elemental material(s) first.\"\n", + " )\n", + "\n", + "for symbol in elements:\n", + " holder = elemental_total_energy_holders[symbol]\n", + " print(\n", + " f\"♻️ Found refined total energy for {symbol}: {holder['_id']} -> \"\n", + " f\"{holder['data']['value']}\"\n", + " )\n", + "\n", + "display_JSON({\"elemental_total_energy_holders\": elemental_total_energy_holders}, level=2)" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "### 4.3. Save compound material for the workflow\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.core.entity.material.api import get_or_create_material\n", + "\n", + "saved_material_response = get_or_create_material(client, material, ACCOUNT_ID)\n", + "saved_material = Material.create(saved_material_response)\n", + "\n", + "print(f\"✅ Saved compound material: {saved_material_response['_id']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "24", + "metadata": {}, + "source": [ + "## 5. Configure the Formation Energy workflow\n", + "### 5.1. Select application\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.ade.application import Application\n", + "from mat3ra.standata.applications import ApplicationStandata\n", + "\n", + "app_config = ApplicationStandata.get_by_name_first_match(APPLICATION_NAME)\n", + "app = Application(**app_config)\n", + "print(f\"Using application: {app.name}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "26", + "metadata": {}, + "source": [ + "### 5.2. Load workflow from Standata and preview it\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.standata.workflows import WorkflowStandata\n", + "from mat3ra.wode.context.providers import PointsGridDataProvider\n", + "from mat3ra.wode.workflows import Workflow\n", + "from mat3ra.notebooks_utils.core.entity.workflow.api import get_or_create_workflow\n", + "from mat3ra.notebooks_utils.ipython.entity.workflow.visualize import visualize_workflow\n", + "\n", + "\n", + "def apply_workflow_kgrids(workflow: Workflow, scf_kgrid=None) -> Workflow:\n", + " if scf_kgrid is not None:\n", + " new_context_scf = PointsGridDataProvider(dimensions=scf_kgrid, isEdited=True).get_context_item_data()\n", + " for subworkflow in workflow.subworkflows:\n", + " unit_names = [unit.name for unit in subworkflow.units]\n", + " if \"pw_scf\" not in unit_names:\n", + " continue\n", + " unit_to_modify_scf = subworkflow.get_unit_by_name(name=\"pw_scf\")\n", + " unit_to_modify_scf.add_context(new_context_scf)\n", + " subworkflow.set_unit(unit_to_modify_scf)\n", + " break\n", + " return workflow\n", + "\n", + "\n", + "formation_workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(\n", + " WORKFLOW_SEARCH_TERM\n", + ")\n", + "formation_workflow = Workflow.create(formation_workflow_config)\n", + "formation_workflow.name = MY_WORKFLOW_NAME\n", + "formation_workflow = apply_workflow_kgrids(formation_workflow, scf_kgrid=SCF_KGRID)\n", + "\n", + "visualize_workflow(formation_workflow)" + ] + }, + { + "cell_type": "markdown", + "id": "28", + "metadata": {}, + "source": [ + "### 5.3. Save workflow to collection\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "saved_formation_workflow_response = get_or_create_workflow(client, formation_workflow, ACCOUNT_ID)\n", + "saved_formation_workflow = Workflow.create(saved_formation_workflow_response)\n", + "print(f\"Formation workflow ID: {saved_formation_workflow.id}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "30", + "metadata": {}, + "source": [ + "## 6. Create the compute configuration\n", + "### 6.1. Select cluster\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31", + "metadata": {}, + "outputs": [], + "source": [ + "clusters = client.clusters.list()\n", + "print(f\"Available clusters: {[c['hostname'] for c in clusters]}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "32", + "metadata": {}, + "source": [ + "### 6.2. Create compute configuration\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.ide.compute import Compute\n", + "\n", + "if CLUSTER_NAME:\n", + " cluster = next((c for c in clusters if CLUSTER_NAME in c[\"hostname\"]), None)\n", + "else:\n", + " cluster = clusters[0]\n", + "\n", + "compute = Compute(cluster=cluster, queue=QUEUE_NAME, ppn=PPN)\n", + "print(f\"Using cluster: {compute.cluster.hostname}, queue: {QUEUE_NAME}, ppn: {PPN}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "34", + "metadata": {}, + "source": [ + "## 7. Create the Formation Energy job\n", + "### 7.1. Create job with compound material and workflow configuration\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.job import create_job\n", + "from mat3ra.notebooks_utils.api.job import wait_for_jobs_to_finish_async\n", + "from mat3ra.utils.namespace import dict_to_namespace_recursive\n", + "\n", + "print(f\"Compound material: {saved_material.id}\")\n", + "print(f\"Elemental references: {', '.join(elements)}\")\n", + "print(f\"Formation workflow: {saved_formation_workflow.id}\")\n", + "print(f\"Project: {project_id}\")\n", + "\n", + "formation_job_name = f\"{MY_WORKFLOW_NAME} {saved_material.formula} {timestamp}\"\n", + "formation_job_response = create_job(\n", + " api_client=client,\n", + " materials=[saved_material],\n", + " workflow=formation_workflow,\n", + " project_id=project_id,\n", + " owner_id=ACCOUNT_ID,\n", + " prefix=formation_job_name,\n", + " compute=compute.to_dict(),\n", + ")\n", + "\n", + "formation_job = dict_to_namespace_recursive(formation_job_response)\n", + "formation_job_id = formation_job._id\n", + "print(f\"✅ Formation Energy job created successfully: {formation_job_id}\")\n", + "display_JSON(formation_job_response)" + ] + }, + { + "cell_type": "markdown", + "id": "36", + "metadata": {}, + "source": [ + "### 7.2. Submit the Formation Energy job and monitor the status\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37", + "metadata": {}, + "outputs": [], + "source": [ + "client.jobs.submit(formation_job_id)\n", + "print(f\"✅ Job {formation_job_id} submitted successfully!\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38", + "metadata": {}, + "outputs": [], + "source": [ + "await wait_for_jobs_to_finish_async(client.jobs, [formation_job_id], poll_interval=POLL_INTERVAL)\n" + ] + }, + { + "cell_type": "markdown", + "id": "39", + "metadata": {}, + "source": [ + "## 8. Retrieve results\n", + "### 8.1. Retrieve and visualize Formation Energy results\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.prode import PropertyName\n", + "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", + "\n", + "formation_energy_data = client.properties.get_for_job(formation_job_id, property_name=\"formation_energy\")\n", + "formation_energy_contributions_data = client.properties.get_for_job(\n", + " formation_job_id,\n", + " property_name=\"formation_energy_contributions\",\n", + ")\n", + "compound_total_energy_data = client.properties.get_for_job(\n", + " formation_job_id,\n", + " property_name=PropertyName.scalar.total_energy.value,\n", + ")\n", + "\n", + "if formation_energy_data:\n", + " visualize_properties(formation_energy_data, title=\"Formation Energy\")\n", + "else:\n", + " print(\"No 'formation_energy' property was returned for the job.\")\n", + "\n", + "if formation_energy_contributions_data:\n", + " visualize_properties(\n", + " formation_energy_contributions_data,\n", + " title=\"Formation Energy Contributions\",\n", + " )\n", + "else:\n", + " print(\"No 'formation_energy_contributions' property was returned for the job.\")\n", + "\n", + "if compound_total_energy_data:\n", + " visualize_properties(compound_total_energy_data, title=\"Compound Total Energy\")\n", + "else:\n", + " print(\"No compound total energy property was returned for the job.\")\n", + "\n", + "print(f\"Compound elements: {elements}\")\n", + "print(f\"Saved compound material used by the workflow: {saved_material_response['_id']}\")\n", + "for symbol in elements:\n", + " holder = elemental_total_energy_holders[symbol]\n", + " print(\n", + " f\"Refined elemental total energy for {symbol}: \"\n", + " f\"{element_materials[symbol]['name']} -> {holder['data']['value']}\"\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}