diff --git a/.gitmodules b/.gitmodules index 6e456528..0479e3d8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -34,3 +34,6 @@ [submodule "Plugins/nosWebRTC/Source/ssl-cert"] path = Plugins/nosWebRTC/Source/ssl-cert url = https://github.com/mediaz/ssl-cert.git +[submodule "Plugins/nosUtilities/External/freetype"] + path = Plugins/nosUtilities/External/freetype + url = https://github.com/freetype/freetype.git diff --git a/Plugins/nosFilters/Config/BokehDof.nosdef b/Plugins/nosFilters/Config/BokehDof.nosdef new file mode 100644 index 00000000..df5b08fb --- /dev/null +++ b/Plugins/nosFilters/Config/BokehDof.nosdef @@ -0,0 +1,121 @@ +{ + "nodes": [ + { + "class_name": "BokehDof", + "menu_info": { + "category": "Filters", + "display_name": "Bokeh DoF" + }, + "node": { + "class_name": "BokehDof", + "name": "Bokeh DoF", + "description": "Single-pass 2D depth-of-field. CoC is computed from a linear view-space Z input; samples are gathered on a Vogel disc weighted by the BokehShape kernel texture, so bokeh takes the shape painted into BokehShape.", + "contents_type": "Job", + "contents": { + "type": "nos.sys.vulkan.GPUNode", + "options": { + "shader": "Shaders/BokehDof.frag" + } + }, + "pins": [ + { + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "LINEAR" + } + }, + { + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "NEAREST" + } + }, + { + "name": "BokehShape", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "LINEAR" + } + }, + { + "name": "FocusDistance", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 5.0, + "min": 0.0, + "max": 1000.0 + }, + { + "name": "FocusRange", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 2.0, + "min": 0.01, + "max": 1000.0 + }, + { + "name": "MaxRadius", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 16.0, + "min": 0.0, + "max": 128.0 + }, + { + "name": "MinRadius", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.5, + "min": 0.0, + "max": 8.0 + }, + { + "name": "BackgroundIsFar", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1.0, + "min": 0.0, + "max": 1.0 + }, + { + "name": "SampleCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 48.0, + "min": 4.0, + "max": 256.0 + }, + { + "name": "KernelRotation", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.0, + "min": -6.2832, + "max": 6.2832 + }, + { + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY" + } + ] + } + } + ] +} diff --git a/Plugins/nosFilters/Config/BokehShape.nosdef b/Plugins/nosFilters/Config/BokehShape.nosdef new file mode 100644 index 00000000..3a466b9a --- /dev/null +++ b/Plugins/nosFilters/Config/BokehShape.nosdef @@ -0,0 +1,93 @@ +{ + "nodes": [ + { + "class_name": "BokehShape", + "menu_info": { + "category": "Filters", + "display_name": "Bokeh Shape" + }, + "node": { + "class_name": "BokehShape", + "name": "Bokeh Shape", + "description": "Procedural bokeh kernel generator. Produces a unit-disc grayscale mask shaped like a regular polygon aperture (blade count, roundness, rotation), with soft edge and optional rim brightening. Feed the Output into a Bokeh DoF node's BokehShape pin.", + "contents_type": "Job", + "contents": { + "type": "nos.sys.vulkan.GPUNode", + "options": { + "shader": "Shaders/BokehShape.frag" + } + }, + "pins": [ + { + "name": "BladeCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 6.0, + "min": 0.0, + "max": 16.0 + }, + { + "name": "Roundness", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.3, + "min": 0.0, + "max": 1.0 + }, + { + "name": "Rotation", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.0, + "min": -6.2832, + "max": 6.2832 + }, + { + "name": "EdgeSoftness", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.04, + "min": 0.0, + "max": 0.5 + }, + { + "name": "RimBoost", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.0, + "min": 0.0, + "max": 4.0 + }, + { + "name": "RimWidth", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.08, + "min": 0.005, + "max": 0.5 + }, + { + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "data": { + "resolution": "CUSTOM", + "width": 128, + "height": 128, + "format": "R16_UNORM", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET", + "filtering": "LINEAR" + } + } + ] + } + } + ] +} diff --git a/Plugins/nosFilters/Config/DepthOfField.nosdef b/Plugins/nosFilters/Config/DepthOfField.nosdef new file mode 100644 index 00000000..1db5a70f --- /dev/null +++ b/Plugins/nosFilters/Config/DepthOfField.nosdef @@ -0,0 +1,990 @@ +{ "nodes": [ + { + "class_name": "nos.filters.DepthOfField", + "node": { + "id": "5899940c-437e-4f71-b119-bb80fb5d1e1a", + "name": "DepthOfField", + "class_name": "nos.filters.DepthOfField", + "pins": [ + { + "id": "1950c2e6-a0f6-485b-8a02-bded8a2f6ed5", + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "PortalPin", + "contents": { "source_id": "74a1bfd0-4f2d-447b-945c-8d0cb67a2120" } + }, + { + "id": "e0b8f433-212f-48f6-ba4f-c8a194e1a707", + "name": "FocusDistance", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 5.0, + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "contents_type": "PortalPin", + "contents": { "source_id": "e709c7b4-9a59-4546-be53-0dc51abc5605" } + }, + { + "id": "68187c92-92f3-40d0-8b24-df6f33f9f649", + "name": "FocusRange", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 2.0, + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "contents_type": "PortalPin", + "contents": { "source_id": "534a26e9-1ebd-4ed2-89fb-bdf5d34b6ec1" } + }, + { + "id": "42554a0a-2d70-4ec4-a2ea-594ad71559f3", + "name": "MaxRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 16.0, + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "contents_type": "PortalPin", + "contents": { "source_id": "63f77504-73aa-4b89-8849-65e27649b272" } + }, + { + "id": "312c4450-a4ad-4690-ba3d-afcbc93da6eb", + "name": "MinRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 0.5, + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "contents_type": "PortalPin", + "contents": { "source_id": "97561978-6da1-4a33-a6bc-c654008a8261" } + }, + { + "id": "ce6c0d45-8ce1-47ef-bd73-addda06d826e", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "PortalPin", + "contents": { "source_id": "c278680b-43b5-40ce-b1af-a4551c2e58f0" } + }, + { + "id": "2be9d3ba-9386-43b0-ae1c-58168be2a289", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED", + "filtering": "LINEAR" + }, + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "PortalPin", + "contents": { "source_id": "479248dc-200d-4a4d-87d2-f2c7c77f667f" } + } + ], + "pos": { "x": 0.0, "y": 0.0 }, + "contents_type": "Graph", + "contents": { "nodes": [ + { + "id": "393281e0-2cb8-4b90-a98e-a8e708719229", + "name": "Output", + "class_name": "nos.internal.GraphOutput", + "pins": [ + { + "id": "2e4ec877-e014-49ae-ae1d-881a0e4d1ac5", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "c278680b-43b5-40ce-b1af-a4551c2e58f0", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "referred_by": [ + "ce6c0d45-8ce1-47ef-bd73-addda06d826e" + ], + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" }, + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 1329.0, "y": 1025.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "6a261add-ff1c-49ba-b9b7-a3bbad8e1fb3", + "name": "Directional DoF (1)", + "class_name": "nos.filters.DirectionalDof", + "pins": [ + { + "id": "b1b03fce-6863-42e6-a78a-260743b5441d", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET", + "filtering": "LINEAR" + }, + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "14aab6c6-10ce-4a39-9c6f-8c5633fe59e2", + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "d6a91b7a-b576-487b-bd2c-89fee90a37d1", + "name": "FocusDistance", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 7.4, + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "55bffdc3-e0fa-4c0f-9ead-5a3b96c232bf", + "name": "FocusRange", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 3.1, + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "bdc5ae5a-10cc-4c3c-b013-573a64bd8ec6", + "name": "MaxRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 5.0, + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "d77d3716-69f5-4c5d-a342-414dc11597fb", + "name": "MinRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 0.0, + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "ad25df82-1942-4f9f-a062-c072261a2d92", + "name": "BackgroundIsFar", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 1.0, + "min": 0.0, + "max": 1.0, + "def": 1.0, + "step": 0.01, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "aaff92e1-63fe-4253-8edb-1f34a76019c9", + "name": "Direction", + "type_name": "nos.fb.vec2", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { "x": 0.0, "y": 1.0 }, + "min": { "x": -1.0, "y": -1.0 }, + "max": { "x": 1.0, "y": 1.0 }, + "def": { "x": 1.0, "y": 0.0 }, + "step": 0.02, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "ad6603e0-2b1d-4bf6-a1d1-af0fc05978a2", + "name": "SampleCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 12.0, + "min": 1.0, + "max": 64.0, + "def": 12.0, + "step": 0.63, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "0ef0f439-9766-4957-8931-a02ce1019bd1", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 1129.0, "y": 1073.0 }, + "contents_type": "Job", + "contents": { "type": "nos.sys.vulkan.GPUNode", "options": { "shader": "Shaders/DirectionalDof.frag" } }, + "function_category": "Default Node", + "description": "1D depth-aware blur. CoC is computed per pixel from a linear view-space Z input. Chain two instances along (1,0) and (0,1) for a separable disc bokeh.", + "plugin_version": { "major": 1, "minor": 7, "patch": 0 } + }, + { + "id": "deac982f-b51b-4ae0-b6c6-9b2998d3e5a9", + "name": "MaxRadius", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "d5387b2e-f8c6-4b2e-8a42-a11eed779a1d", + "name": "Output", + "type_name": "float", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": 5.0, + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "63f77504-73aa-4b89-8849-65e27649b272", + "name": "Input", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 16.0, + "referred_by": [ + "42554a0a-2d70-4ec4-a2ea-594ad71559f3" + ], + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "meta_data_map": [ + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1250.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "af576b2d-dde0-4d7b-86fc-37cb9f97b49e", + "name": "Directional DoF", + "class_name": "nos.filters.DirectionalDof", + "pins": [ + { + "id": "9e368dde-bb31-44d8-aaad-782e92fe2366", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED", + "filtering": "LINEAR" + }, + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "7e88a91a-1eca-4cc5-8dce-7c4aca61368d", + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "c1b814c4-e424-40a0-99d6-0437d948d1d7", + "name": "FocusDistance", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 7.4, + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "78893474-3dfc-4a36-b897-77760ba19c8c", + "name": "FocusRange", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 3.1, + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "4f471215-bebf-49be-a6e4-909c394d1f1a", + "name": "MaxRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 5.0, + "min": 0.0, + "max": 128.0, + "def": 16.0, + "step": 1.28, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "05132381-cf95-4253-9fc2-e87f84b70dd8", + "name": "MinRadius", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 0.0, + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "2b52da00-b45d-41ae-a1ec-c88566879043", + "name": "BackgroundIsFar", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 1.0, + "min": 0.0, + "max": 1.0, + "def": 1.0, + "step": 0.01, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "ac684e31-1a90-462f-8b64-2b368a93b563", + "name": "Direction", + "type_name": "nos.fb.vec2", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { "x": 1.0, "y": 0.0 }, + "min": { "x": -1.0, "y": -1.0 }, + "max": { "x": 1.0, "y": 1.0 }, + "def": { "x": 1.0, "y": 0.0 }, + "step": 0.02, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "b8527f03-5c5c-4a41-b485-fa05e0f50cb1", + "name": "SampleCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 12.0, + "min": 1.0, + "max": 64.0, + "def": 12.0, + "step": 0.63, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "8f4d23a7-3b94-4a1c-ba14-d1ce47e92acd", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "def": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED STORAGE RENDER_TARGET" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 855.0, "y": 977.0 }, + "contents_type": "Job", + "contents": { "type": "nos.sys.vulkan.GPUNode", "options": { "shader": "Shaders/DirectionalDof.frag" } }, + "function_category": "Default Node", + "description": "1D depth-aware blur. CoC is computed per pixel from a linear view-space Z input. Chain two instances along (1,0) and (0,1) for a separable disc bokeh.", + "plugin_version": { "major": 1, "minor": 7, "patch": 0 } + }, + { + "id": "8b497dab-5466-4d32-a440-125976e3a3ee", + "name": "Depth", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "9587b7b1-8fc7-437b-9459-ee73f90de097", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "74a1bfd0-4f2d-447b-945c-8d0cb67a2120", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED" + }, + "referred_by": [ + "1950c2e6-a0f6-485b-8a02-bded8a2f6ed5" + ], + "def": { + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" }, + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1025.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "9813ee9d-1f75-4554-9f9c-b9ecafc2e9fe", + "name": "FocusDistance", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "7c60934b-ba19-4faf-9923-411511649cd0", + "name": "Output", + "type_name": "float", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": 7.4, + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "e709c7b4-9a59-4546-be53-0dc51abc5605", + "name": "Input", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 5.0, + "referred_by": [ + "e0b8f433-212f-48f6-ba4f-c8a194e1a707" + ], + "min": 0.0, + "max": 1000.0, + "def": 5.0, + "step": 10.0, + "meta_data_map": [ + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1100.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "2c0861b9-e416-4741-b56d-8dfa81c49516", + "name": "FocusRange", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "6a933bab-7bf6-4388-b990-abd1b9729e64", + "name": "Output", + "type_name": "float", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": 3.1, + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "534a26e9-1ebd-4ed2-89fb-bdf5d34b6ec1", + "name": "Input", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 2.0, + "referred_by": [ + "68187c92-92f3-40d0-8b24-df6f33f9f649" + ], + "min": 0.01, + "max": 1000.0, + "def": 2.0, + "step": 9.9999, + "meta_data_map": [ + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1175.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "3951aaae-16df-4b07-b1a9-b8b2a01b19c7", + "name": "MinRadius", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "f0feee29-3782-49fe-a834-94e2b57916a8", + "name": "Output", + "type_name": "float", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": 0.0, + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "97561978-6da1-4a33-a6bc-c654008a8261", + "name": "Input", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": 0.5, + "referred_by": [ + "312c4450-a4ad-4690-ba3d-afcbc93da6eb" + ], + "min": 0.0, + "max": 8.0, + "def": 0.5, + "step": 0.08, + "meta_data_map": [ + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1325.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + }, + { + "id": "a1d16ddd-0144-4daa-97b2-e9b3b019c8c1", + "name": "Input", + "class_name": "nos.internal.GraphInput", + "pins": [ + { + "id": "c271ac23-2923-45c2-b262-b654455a93c3", + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED", + "filtering": "LINEAR" + }, + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + }, + { + "id": "479248dc-200d-4a4d-87d2-f2c7c77f667f", + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + }, + "data": { + "resolution": "HD", + "width": 1920, + "height": 1080, + "format": "R16G16B16A16_SFLOAT", + "usage": "TRANSFER_SRC TRANSFER_DST SAMPLED", + "filtering": "LINEAR" + }, + "referred_by": [ + "2be9d3ba-9386-43b0-ae1c-58168be2a289" + ], + "def": { + "filtering": "LINEAR" + }, + "advanced_property": true, + "meta_data_map": [ + { "key": "AdvancedProperty", "value": "true" }, + { "key": "PinHidden", "value": "true" } + ], + "contents_type": "JobPin", + "contents": { } + } + ], + "pos": { "x": 655.0, "y": 1400.0 }, + "contents_type": "Job", + "contents": { "type": "" }, + "function_category": "Default Node", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + } + ], "connections": [ + { "from": "8f4d23a7-3b94-4a1c-ba14-d1ce47e92acd", "to": "b1b03fce-6863-42e6-a78a-260743b5441d", "id": "83839676-0760-4699-ae80-c0a789e273d8" }, + { "from": "f0feee29-3782-49fe-a834-94e2b57916a8", "to": "d77d3716-69f5-4c5d-a342-414dc11597fb", "id": "4c05135f-6001-4679-b39c-b248559ae56d" }, + { "from": "9587b7b1-8fc7-437b-9459-ee73f90de097", "to": "14aab6c6-10ce-4a39-9c6f-8c5633fe59e2", "id": "231cdfe5-7ac7-4013-9d20-68d5af8509b7" }, + { "from": "7c60934b-ba19-4faf-9923-411511649cd0", "to": "d6a91b7a-b576-487b-bd2c-89fee90a37d1", "id": "1cdaef73-876c-472a-97ff-04bf1f01348e" }, + { "from": "c271ac23-2923-45c2-b262-b654455a93c3", "to": "9e368dde-bb31-44d8-aaad-782e92fe2366", "id": "231fc88c-a52e-48d0-a6ee-8c2fdfe3ef0d" }, + { "from": "6a933bab-7bf6-4388-b990-abd1b9729e64", "to": "55bffdc3-e0fa-4c0f-9ead-5a3b96c232bf", "id": "7c1cae59-5834-420e-9d3d-e4767f6c3273" }, + { "from": "d5387b2e-f8c6-4b2e-8a42-a11eed779a1d", "to": "bdc5ae5a-10cc-4c3c-b013-573a64bd8ec6", "id": "d74bdb3a-8c8c-4f82-8038-01a237e27a89" }, + { "from": "0ef0f439-9766-4957-8931-a02ce1019bd1", "to": "2e4ec877-e014-49ae-ae1d-881a0e4d1ac5", "id": "353cc954-d098-417a-8331-357b879ba654" }, + { "from": "9587b7b1-8fc7-437b-9459-ee73f90de097", "to": "7e88a91a-1eca-4cc5-8dce-7c4aca61368d", "id": "b126f4c4-d748-46f2-be51-ce1c778c0c4b" }, + { "from": "7c60934b-ba19-4faf-9923-411511649cd0", "to": "c1b814c4-e424-40a0-99d6-0437d948d1d7", "id": "fc25a2f4-0af4-49ae-9052-133a76cfc044" }, + { "from": "6a933bab-7bf6-4388-b990-abd1b9729e64", "to": "78893474-3dfc-4a36-b897-77760ba19c8c", "id": "f6ba18f8-0ef1-42db-a774-c4b02aa78fac" }, + { "from": "d5387b2e-f8c6-4b2e-8a42-a11eed779a1d", "to": "4f471215-bebf-49be-a6e4-909c394d1f1a", "id": "afd9d7ff-f9e2-4a67-b874-2cfb2f870447" }, + { "from": "f0feee29-3782-49fe-a834-94e2b57916a8", "to": "05132381-cf95-4253-9fc2-e87f84b70dd8", "id": "82919455-4a51-490a-8ab2-201952d2e126" } + ] }, + "function_category": "Default Node", + "display_name": "Depth of Field", + "plugin_version": { "major": 0, "minor": 0, "patch": 0 } + } + } + ] } diff --git a/Plugins/nosFilters/Config/DirectionalDof.nosdef b/Plugins/nosFilters/Config/DirectionalDof.nosdef new file mode 100644 index 00000000..427e4385 --- /dev/null +++ b/Plugins/nosFilters/Config/DirectionalDof.nosdef @@ -0,0 +1,121 @@ +{ + "nodes": [ + { + "class_name": "DirectionalDof", + "menu_info": { + "category": "Filters", + "display_name": "Directional DoF" + }, + "node": { + "class_name": "DirectionalDof", + "name": "Directional DoF", + "description": "1D depth-aware blur. CoC is computed per pixel from a linear view-space Z input. Chain two instances along (1,0) and (0,1) for a separable disc bokeh.", + "contents_type": "Job", + "contents": { + "type": "nos.sys.vulkan.GPUNode", + "options": { + "shader": "Shaders/DirectionalDof.frag" + } + }, + "pins": [ + { + "name": "Input", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "LINEAR" + } + }, + { + "name": "Depth", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "filtering": "NEAREST" + } + }, + { + "name": "FocusDistance", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 5.0, + "min": 0.0, + "max": 1000.0 + }, + { + "name": "FocusRange", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 2.0, + "min": 0.01, + "max": 1000.0 + }, + { + "name": "MaxRadius", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 16.0, + "min": 0.0, + "max": 128.0 + }, + { + "name": "MinRadius", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0.5, + "min": 0.0, + "max": 8.0 + }, + { + "name": "BackgroundIsFar", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1.0, + "min": 0.0, + "max": 1.0 + }, + { + "name": "Direction", + "type_name": "nos.fb.vec2", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "x": 1.0, + "y": 0.0 + }, + "min": { + "x": -1.0, + "y": -1.0 + }, + "max": { + "x": 1.0, + "y": 1.0 + } + }, + { + "name": "SampleCount", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 12.0, + "min": 1.0, + "max": 64.0 + }, + { + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY" + } + ] + } + } + ] +} diff --git a/Plugins/nosFilters/Filters.noscfg b/Plugins/nosFilters/Filters.noscfg index 072c6ce0..3660bb3f 100644 --- a/Plugins/nosFilters/Filters.noscfg +++ b/Plugins/nosFilters/Filters.noscfg @@ -2,7 +2,7 @@ "info": { "id": { "name": "nos.filters", - "version": "1.7.0" + "version": "1.8.0" }, "display_name": "Filters", "description": "Collection of image filters.", @@ -27,6 +27,10 @@ "Config/Diff.nosdef", "Config/GaussianBlur.nosdef", "Config/DirectionalBlur.nosdef", + "Config/DirectionalDof.nosdef", + "Config/DepthOfField.nosdef", + "Config/BokehDof.nosdef", + "Config/BokehShape.nosdef", "Config/KawaseLightStreak.nosdef", "Config/Kuwahara.nosdef", "Config/PremultiplyAlpha.nosdef", diff --git a/Plugins/nosFilters/Shaders/BokehDof.frag b/Plugins/nosFilters/Shaders/BokehDof.frag new file mode 100644 index 00000000..b365ddf2 --- /dev/null +++ b/Plugins/nosFilters/Shaders/BokehDof.frag @@ -0,0 +1,105 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. +// Single-pass 2D bokeh depth-of-field with a kernel-texture shaping the bokeh. +// +// Computes a per-pixel circle of confusion (CoC) from a linear view-space Z +// input, then gathers samples on a Vogel (golden-angle) disc within that CoC. +// Each sample's contribution is weighted by BokehShape sampled at the same +// unit-disc position, so the bokeh takes on the shape painted into BokehShape +// (regular polygon, ring, custom artwork, etc.). + +#version 450 + +#define MASK_THRESHOLD 0.001 +#define GOLDEN_ANGLE 2.39996322972865332 + +layout(binding = 0) uniform sampler2D Input; +layout(binding = 1) uniform sampler2D Depth; +layout(binding = 2) uniform sampler2D BokehShape; +layout(binding = 3) uniform BokehDofParams +{ + // Focus distance in the same units as the Depth input (linear view-space Z). + float FocusDistance; + // Distance from focus where CoC reaches MaxRadius. + float FocusRange; + // Maximum CoC radius in pixels. + float MaxRadius; + // Skip the gather when CoC <= MinRadius (keeps focused regions crisp & cheap). + float MinRadius; + // 0 = treat zero depth as "near focus" (stays sharp); 1 = treat as far plane. + float BackgroundIsFar; + // Total Vogel-disc sample count. ~32 = soft, ~64 = clean, ~128 = no banding. + float SampleCount; + // Rotate the kernel lookup (radians). Useful for animated highlights. + float KernelRotation; +} +Params; + +layout(location = 0) out vec4 rt; +layout(location = 0) in vec2 uv; + +float CocFromDepth(float Z) +{ + if (Z <= 0.0) + Z = mix(Params.FocusDistance, Params.FocusDistance + Params.FocusRange * 4.0, Params.BackgroundIsFar); + + float D = abs(Z - Params.FocusDistance); + float Coc = D / max(Params.FocusRange, 1e-4); + return clamp(Coc * Params.MaxRadius, 0.0, Params.MaxRadius); +} + +void main() +{ + vec2 TextureSize = textureSize(Input, 0); + vec2 TexelSize = 1.0 / TextureSize; + + vec4 CenterColor = texture(Input, uv); + float CenterZ = texture(Depth, uv).r; + float CenterCoC = CocFromDepth(CenterZ); + + if (CenterCoC <= Params.MinRadius || Params.MaxRadius < MASK_THRESHOLD) + { + rt = CenterColor; + return; + } + + int N = int(max(1.0, Params.SampleCount)); + float CosR = cos(Params.KernelRotation); + float SinR = sin(Params.KernelRotation); + + // Vogel disc: golden-angle spiral with sqrt radius for uniform area density. + // Sample 0 is the center; included implicitly via CenterColor initialization. + vec4 Accum = CenterColor; + float Weight = texture(BokehShape, vec2(0.5)).r; + Accum *= Weight; + + for (int i = 1; i < N; ++i) + { + float Frac = float(i) / float(N); + float R = sqrt(Frac); // unit-disc radius + float Th = float(i) * GOLDEN_ANGLE; + vec2 Unit = vec2(cos(Th) * R, sin(Th) * R); // unit disc position + + // Rotated lookup into the bokeh kernel. + vec2 ShapeUv = vec2(Unit.x * CosR - Unit.y * SinR, + Unit.x * SinR + Unit.y * CosR) * 0.5 + 0.5; + float WShape = texture(BokehShape, ShapeUv).r; + if (WShape <= MASK_THRESHOLD) + continue; + + vec2 Ofs = Unit * CenterCoC * TexelSize; + vec4 Sample = texture(Input, uv + Ofs); + float ZSamp = texture(Depth, uv + Ofs).r; + float CocSmp = CocFromDepth(ZSamp); + + // Per-sample CoC rejection prevents in-focus pixels bleeding outward. + // A sample contributes only if its own CoC is at least its distance from center. + float Dist = R * CenterCoC; + float WCoc = Dist <= CocSmp ? 1.0 : 0.0; + + float W = WShape * WCoc; + Accum += Sample * W; + Weight += W; + } + + rt = Accum / max(Weight, 1e-4); +} diff --git a/Plugins/nosFilters/Shaders/BokehShape.frag b/Plugins/nosFilters/Shaders/BokehShape.frag new file mode 100644 index 00000000..cb963629 --- /dev/null +++ b/Plugins/nosFilters/Shaders/BokehShape.frag @@ -0,0 +1,77 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. +// Procedural bokeh kernel generator. +// +// Produces a grayscale unit-disc mask shaped like a regular polygon aperture +// (number of blades configurable) with optional roundness, rotation, soft edge +// and brightened rim. Intended as input to a kernel-weighted DoF gather. +// +// Convention: image is treated as the [-1, 1] unit square; pixels outside the +// kernel shape return 0; pixels inside return ~1, with a smooth edge falloff +// over EdgeSoftness. The mask is normalized so that center stays at 1. + +#version 450 + +#define PI 3.14159265358979323846 + +layout(location = 0) out vec4 rt; +layout(location = 0) in vec2 uv; + +layout(binding = 1) uniform BokehShapeParams +{ + // Aperture blade count. 0 or 1 = perfect circle. + float BladeCount; + // 0 = sharp polygon, 1 = perfect circle. Interpolates polygon edge toward disc. + float Roundness; + // Rotation of the polygon (radians). + float Rotation; + // Soft falloff width at the edge, in [0, 1] of unit-disc radius. + float EdgeSoftness; + // Extra brightness boost near the rim, [0, 1]. Mimics cat's-eye / specular bokeh. + float RimBoost; + // Width of the rim brightening band, in [0, 1] of radius. + float RimWidth; +} +Params; + +void main() +{ + // Map uv [0,1] to centered coords [-1,1] + vec2 Pos = uv * 2.0 - 1.0; + float R = length(Pos); + + if (R > 1.0) + { + rt = vec4(0.0); + return; + } + + float Blades = max(Params.BladeCount, 1.0); + + // Polygon edge radius along this angular direction. + // sectorAngle = 2*pi / N; angle from sector center is a; edge distance = cos(pi/N) / cos(a). + float PolygonR = 1.0; + if (Blades >= 3.0) + { + float Theta = atan(Pos.y, Pos.x) - Params.Rotation; + float SectorAngle = 2.0 * PI / Blades; + float HalfSector = SectorAngle * 0.5; + // Angle measured from the nearest sector centerline, in [-HalfSector, +HalfSector]. + float A = mod(Theta + HalfSector, SectorAngle) - HalfSector; + PolygonR = cos(HalfSector) / max(cos(A), 1e-4); + } + + // Roundness mixes polygon edge toward the circumscribed circle (radius 1). + float EdgeR = mix(PolygonR, 1.0, clamp(Params.Roundness, 0.0, 1.0)); + + // Soft edge: 1 inside, 0 past the edge, smooth across EdgeSoftness. + float Soft = max(Params.EdgeSoftness, 1e-4); + float Mask = 1.0 - smoothstep(EdgeR - Soft, EdgeR, R); + + // Rim brightening: a soft band just inside the edge. + float RimW = max(Params.RimWidth, 1e-4); + float RimPos = (R - (EdgeR - RimW)) / RimW; // 0 at inner edge of rim, 1 at outer + float Rim = clamp(1.0 - abs(RimPos * 2.0 - 1.0), 0.0, 1.0); + Mask += Rim * Params.RimBoost * Mask; + + rt = vec4(Mask, Mask, Mask, 1.0); +} diff --git a/Plugins/nosFilters/Shaders/DirectionalDof.frag b/Plugins/nosFilters/Shaders/DirectionalDof.frag new file mode 100644 index 00000000..308cfcc7 --- /dev/null +++ b/Plugins/nosFilters/Shaders/DirectionalDof.frag @@ -0,0 +1,95 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. +// Directional depth-of-field pass. +// Computes circle-of-confusion (CoC) per pixel from a linear view-space Z input, +// then does a 1D weighted gather along Direction. Chain two instances +// (Direction = (1,0) and Direction = (0,1)) for a separable approximation of +// disc bokeh; visually close to a gaussian bokeh and cheap. + +#version 450 + +#define MASK_THRESHOLD 0.001 + +layout(binding = 0) uniform sampler2D Input; +layout(binding = 1) uniform sampler2D Depth; +layout(binding = 2) uniform DirectionalDofParams +{ + // Focus distance in the same units as the Depth input (linear view-space Z). + float FocusDistance; + // Distance from focus where CoC reaches MaxRadius. + // Smaller value = sharper focus falloff; larger = gentler. + float FocusRange; + // Maximum CoC radius in pixels. + float MaxRadius; + // 0 = treat zero depth as "no info, keep sharp"; 1 = treat zero depth as far. + float BackgroundIsFar; + vec2 Direction; + // Optional: clamp CoC near the focus plane to avoid noise; raise to skip tiny blurs. + float MinRadius; + // Sample count along the direction (one side; total taps = 2*N+1). Higher = smoother. + float SampleCount; +} +Params; + +layout(location = 0) out vec4 rt; +layout(location = 0) in vec2 uv; + +float CocFromDepth(float Z) +{ + // Treat Z<=0 (no depth signal) as either "near focus" (BackgroundIsFar=0) + // or as far plane (BackgroundIsFar=1). Picking far avoids halos around empty regions. + if (Z <= 0.0) + Z = mix(Params.FocusDistance, Params.FocusDistance + Params.FocusRange * 4.0, Params.BackgroundIsFar); + + float D = abs(Z - Params.FocusDistance); + float Coc = D / max(Params.FocusRange, 1e-4); + Coc = clamp(Coc * Params.MaxRadius, 0.0, Params.MaxRadius); + return Coc; +} + +void main() +{ + vec2 TextureSize = textureSize(Input, 0); + vec2 TexelSize = 1.0 / TextureSize; + + vec4 CenterColor = texture(Input, uv); + float CenterZ = texture(Depth, uv).r; + float CenterCoC = CocFromDepth(CenterZ); + + if (CenterCoC <= Params.MinRadius || Params.MaxRadius < MASK_THRESHOLD) + { + rt = CenterColor; + return; + } + + vec2 Dir = normalize(Params.Direction); + + int N = int(max(1.0, Params.SampleCount)); + float RadiusPx = CenterCoC; + float Step = RadiusPx / float(N); + + // Box-weighted average; for separable-2D this gives a soft disc. + // CoC-clamping per sample prevents fragments in focus from bleeding outward. + vec4 Accum = CenterColor; + float Weight = 1.0; + + for (int i = 1; i <= N; ++i) + { + float T = float(i) * Step; + vec2 Ofs = Dir * T * TexelSize; + + vec4 SPos = texture(Input, uv + Ofs); + float ZPos = texture(Depth, uv + Ofs).r; + float CocPos = CocFromDepth(ZPos); + float WPos = Step <= CocPos ? 1.0 : 0.0; + + vec4 SNeg = texture(Input, uv - Ofs); + float ZNeg = texture(Depth, uv - Ofs).r; + float CocNeg = CocFromDepth(ZNeg); + float WNeg = Step <= CocNeg ? 1.0 : 0.0; + + Accum += SPos * WPos + SNeg * WNeg; + Weight += WPos + WNeg; + } + + rt = Accum / Weight; +} diff --git a/Plugins/nosMath/CMakeLists.txt b/Plugins/nosMath/CMakeLists.txt index ee9ef718..bcbf2b2f 100644 --- a/Plugins/nosMath/CMakeLists.txt +++ b/Plugins/nosMath/CMakeLists.txt @@ -6,6 +6,10 @@ add_library(tinyexpr_cpp STATIC ${TINYEXPR_SOURCES}) target_include_directories(tinyexpr_cpp PUBLIC External/tinyexpr-cpp) nos_group_targets("tinyexpr_cpp" "External") -set(DEPENDENCIES ${NOS_PLUGIN_SDK_TARGET} tinyexpr_cpp) +set(GENERATED_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/Generated") +nos_generate_flatbuffers("${CMAKE_CURRENT_SOURCE_DIR}/Config" "${GENERATED_OUTPUT_DIR}" "cpp" "${NOS_SDK_DIR}/Types" nosMath_generated) -nos_add_plugin("nosMath" "${DEPENDENCIES}" "") +set(DEPENDENCIES ${NOS_PLUGIN_SDK_TARGET} tinyexpr_cpp nosMath_generated) +set(INCLUDE_FOLDERS "${GENERATED_OUTPUT_DIR}") + +nos_add_plugin("nosMath" "${DEPENDENCIES}" "${INCLUDE_FOLDERS}") diff --git a/Plugins/nosMath/Config/EulerToQuaternion.nosdef b/Plugins/nosMath/Config/EulerToQuaternion.nosdef new file mode 100644 index 00000000..fa40e5d7 --- /dev/null +++ b/Plugins/nosMath/Config/EulerToQuaternion.nosdef @@ -0,0 +1,38 @@ +{ + "nodes": [ + { + "class_name": "EulerToQuaternion", + "menu_info": { + "category": "Math|Linear Algebra", + "display_name": "Euler To Quaternion" + }, + "node": { + "class_name": "EulerToQuaternion", + "contents_type": "Job", + "description": "Converts an Euler-angle rotation (degrees) to a unit quaternion (x, y, z, w). The Order pin selects the intrinsic rotation order applied to the components of the input vec3 (e.g. ZYX means R = Rz(rot.z) * Ry(rot.y) * Rx(rot.x)). Default ZYX matches the FreeD/Track convention.", + "pins": [ + { + "name": "Euler", + "type_name": "nos.fb.vec3d", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Order", + "type_name": "nos.math.EulerOrder", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "ZYX", + "description": "Euler intrinsic rotation order applied to the (rot.x, rot.y, rot.z) components." + }, + { + "name": "Quaternion", + "type_name": "nos.fb.vec4d", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY" + } + ] + } + } + ] +} diff --git a/Plugins/nosMath/Config/Math.fbs b/Plugins/nosMath/Config/Math.fbs new file mode 100644 index 00000000..84c79eeb --- /dev/null +++ b/Plugins/nosMath/Config/Math.fbs @@ -0,0 +1,10 @@ +namespace nos.math; + +enum EulerOrder : ubyte { + ZYX = 0, + XYZ = 1, + YXZ = 2, + YZX = 3, + ZXY = 4, + XZY = 5, +} diff --git a/Plugins/nosMath/Config/QuaternionMultiply.nosdef b/Plugins/nosMath/Config/QuaternionMultiply.nosdef new file mode 100644 index 00000000..fc162b41 --- /dev/null +++ b/Plugins/nosMath/Config/QuaternionMultiply.nosdef @@ -0,0 +1,36 @@ +{ + "nodes": [ + { + "class_name": "QuaternionMultiply", + "menu_info": { + "category": "Math|Linear Algebra", + "display_name": "Quaternion Multiply" + }, + "node": { + "class_name": "QuaternionMultiply", + "contents_type": "Job", + "description": "Hamilton product of two unit quaternions: Result = A * B (each as (x, y, z, w)). Composing rotations: A * B applies B first, then A. To rotate (conjugate) a quaternion Q by R, compute R * Q * conjugate(R).", + "pins": [ + { + "name": "A", + "type_name": "nos.fb.vec4d", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "B", + "type_name": "nos.fb.vec4d", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Result", + "type_name": "nos.fb.vec4d", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY" + } + ] + } + } + ] +} diff --git a/Plugins/nosMath/Config/QuaternionToEuler.nosdef b/Plugins/nosMath/Config/QuaternionToEuler.nosdef new file mode 100644 index 00000000..d08ffc7a --- /dev/null +++ b/Plugins/nosMath/Config/QuaternionToEuler.nosdef @@ -0,0 +1,38 @@ +{ + "nodes": [ + { + "class_name": "QuaternionToEuler", + "menu_info": { + "category": "Math|Linear Algebra", + "display_name": "Quaternion To Euler" + }, + "node": { + "class_name": "QuaternionToEuler", + "contents_type": "Job", + "description": "Converts a unit quaternion (x, y, z, w) to Euler angles (degrees). The Order pin selects the intrinsic rotation order extracted (e.g. ZYX yields rot.z = first rotation about Z, rot.y about Y, rot.x about X). Inverse of EulerToQuaternion when the same Order is used on both ends.", + "pins": [ + { + "name": "Quaternion", + "type_name": "nos.fb.vec4d", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Order", + "type_name": "nos.math.EulerOrder", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "ZYX", + "description": "Euler intrinsic rotation order extracted into the (rot.x, rot.y, rot.z) output components." + }, + { + "name": "Euler", + "type_name": "nos.fb.vec3d", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY" + } + ] + } + } + ] +} diff --git a/Plugins/nosMath/Math.noscfg b/Plugins/nosMath/Math.noscfg index de4791ee..5417aca6 100644 --- a/Plugins/nosMath/Math.noscfg +++ b/Plugins/nosMath/Math.noscfg @@ -14,7 +14,9 @@ } ] }, - "custom_types": [], + "custom_types": [ + "Config/Math.fbs" + ], "node_definitions": [ "Config/Math.nosdef", "Config/Eval.nosdef", @@ -25,7 +27,10 @@ "Config/Random.nosdef", "Config/Lerp.nosdef", "Config/Vec3ToVec4.nosdef", - "Config/EmbedMat3ToMat4.nosdef" + "Config/EmbedMat3ToMat4.nosdef", + "Config/EulerToQuaternion.nosdef", + "Config/QuaternionToEuler.nosdef", + "Config/QuaternionMultiply.nosdef" ], "binary_path": "Binaries/nosMath", "third_party_software": [ diff --git a/Plugins/nosMath/Source/EulerToQuaternion.cpp b/Plugins/nosMath/Source/EulerToQuaternion.cpp new file mode 100644 index 00000000..94440a60 --- /dev/null +++ b/Plugins/nosMath/Source/EulerToQuaternion.cpp @@ -0,0 +1,97 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include +#include +#include +#include + +namespace nos::math +{ + +// Build a rotation matrix for the given intrinsic Euler order. +// In all cases, rot.x is the angle about X, rot.y about Y, rot.z about Z (radians). +// Order ZYX means R = Rz(rot.z) * Ry(rot.y) * Rx(rot.x), applied right-to-left to a point. +static glm::dmat4 EulerToMat(EulerOrder order, glm::dvec3 const& r) +{ + switch (order) + { + case EulerOrder::ZYX: return glm::eulerAngleZYX(r.z, r.y, r.x); + case EulerOrder::XYZ: return glm::eulerAngleXYZ(r.x, r.y, r.z); + case EulerOrder::YXZ: return glm::eulerAngleYXZ(r.y, r.x, r.z); + case EulerOrder::YZX: return glm::eulerAngleYZX(r.y, r.z, r.x); + case EulerOrder::ZXY: return glm::eulerAngleZXY(r.z, r.x, r.y); + case EulerOrder::XZY: return glm::eulerAngleXZY(r.x, r.z, r.y); + } + return glm::dmat4(1.0); +} + +static void MatToEuler(EulerOrder order, glm::dmat4 const& m, glm::dvec3& r) +{ + switch (order) + { + case EulerOrder::ZYX: glm::extractEulerAngleZYX(m, r.z, r.y, r.x); break; + case EulerOrder::XYZ: glm::extractEulerAngleXYZ(m, r.x, r.y, r.z); break; + case EulerOrder::YXZ: glm::extractEulerAngleYXZ(m, r.y, r.x, r.z); break; + case EulerOrder::YZX: glm::extractEulerAngleYZX(m, r.y, r.z, r.x); break; + case EulerOrder::ZXY: glm::extractEulerAngleZXY(m, r.z, r.x, r.y); break; + case EulerOrder::XZY: glm::extractEulerAngleXZY(m, r.x, r.z, r.y); break; + } +} + +struct EulerToQuaternionNodeContext : NodeContext +{ + using NodeContext::NodeContext; + + nosResult ExecuteNode(nosNodeExecuteParams* execParams) override + { + nos::NodeExecuteParams params(execParams); + auto* in = params.GetPinData(NOS_NAME("Euler")); + auto* order = params.GetPinData(NOS_NAME("Order")); + auto* out = params.GetPinData(NOS_NAME("Quaternion")); + + glm::dvec3 r = glm::radians(glm::dvec3(in->x(), in->y(), in->z())); + glm::dquat q(EulerToMat(*order, r)); + + out->mutate_x(q.x); + out->mutate_y(q.y); + out->mutate_z(q.z); + out->mutate_w(q.w); + return NOS_RESULT_SUCCESS; + } +}; + +void RegisterEulerToQuaternion(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("nos.math.EulerToQuaternion"), EulerToQuaternionNodeContext, fn) +} + +struct QuaternionToEulerNodeContext : NodeContext +{ + using NodeContext::NodeContext; + + nosResult ExecuteNode(nosNodeExecuteParams* execParams) override + { + nos::NodeExecuteParams params(execParams); + auto* in = params.GetPinData(NOS_NAME("Quaternion")); + auto* order = params.GetPinData(NOS_NAME("Order")); + auto* out = params.GetPinData(NOS_NAME("Euler")); + + glm::dquat q(in->w(), in->x(), in->y(), in->z()); + glm::dmat4 m = glm::mat4_cast(q); + glm::dvec3 r(0.0); + MatToEuler(*order, m, r); + + out->mutate_x(glm::degrees(r.x)); + out->mutate_y(glm::degrees(r.y)); + out->mutate_z(glm::degrees(r.z)); + return NOS_RESULT_SUCCESS; + } +}; + +void RegisterQuaternionToEuler(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("nos.math.QuaternionToEuler"), QuaternionToEulerNodeContext, fn) +} + +} // namespace nos::math diff --git a/Plugins/nosMath/Source/Math.cpp b/Plugins/nosMath/Source/Math.cpp index c307f2cf..1893cc8e 100644 --- a/Plugins/nosMath/Source/Math.cpp +++ b/Plugins/nosMath/Source/Math.cpp @@ -104,6 +104,9 @@ enum class MathNodeTypes : int { Or, Not, Random, + EulerToQuaternion, + QuaternionToEuler, + QuaternionMultiply, Count }; @@ -168,6 +171,9 @@ void RegisterAnd(nosNodeFunctions*); void RegisterOr(nosNodeFunctions*); void RegisterNot(nosNodeFunctions*); void RegisterRandom(nosNodeFunctions*); +void RegisterEulerToQuaternion(nosNodeFunctions*); +void RegisterQuaternionToEuler(nosNodeFunctions*); +void RegisterQuaternionMultiply(nosNodeFunctions*); nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outCount, nosNodeFunctions** outList) { @@ -281,6 +287,18 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outCount, nosNodeFunctions** o RegisterRandom(node); break; } + case MathNodeTypes::EulerToQuaternion: { + RegisterEulerToQuaternion(node); + break; + } + case MathNodeTypes::QuaternionToEuler: { + RegisterQuaternionToEuler(node); + break; + } + case MathNodeTypes::QuaternionMultiply: { + RegisterQuaternionMultiply(node); + break; + } default: break; } diff --git a/Plugins/nosMath/Source/QuaternionMultiply.cpp b/Plugins/nosMath/Source/QuaternionMultiply.cpp new file mode 100644 index 00000000..395b4843 --- /dev/null +++ b/Plugins/nosMath/Source/QuaternionMultiply.cpp @@ -0,0 +1,39 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include +#include +#include + +namespace nos::math +{ + +struct QuaternionMultiplyNodeContext : NodeContext +{ + using NodeContext::NodeContext; + + nosResult ExecuteNode(nosNodeExecuteParams* execParams) override + { + nos::NodeExecuteParams params(execParams); + auto* a = params.GetPinData(NOS_NAME("A")); + auto* b = params.GetPinData(NOS_NAME("B")); + auto* out = params.GetPinData(NOS_NAME("Result")); + + glm::dquat qa(a->w(), a->x(), a->y(), a->z()); + glm::dquat qb(b->w(), b->x(), b->y(), b->z()); + glm::dquat qr = qa * qb; + + out->mutate_x(qr.x); + out->mutate_y(qr.y); + out->mutate_z(qr.z); + out->mutate_w(qr.w); + return NOS_RESULT_SUCCESS; + } +}; + +void RegisterQuaternionMultiply(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("nos.math.QuaternionMultiply"), QuaternionMultiplyNodeContext, fn) +} + +} // namespace nos::math diff --git a/Plugins/nosTrack/CHANGES.md b/Plugins/nosTrack/CHANGES.md new file mode 100644 index 00000000..ec713c45 --- /dev/null +++ b/Plugins/nosTrack/CHANGES.md @@ -0,0 +1,57 @@ +# Record Track (COLMAP) Node + +## Summary + +A new node "Record Track (COLMAP)" added to the `nosTrack` plugin. It records incoming camera tracking data per frame and exports it in COLMAP's text format (`cameras.txt` + `images.txt`). + +## Files Changed + +### New files +- `Source/RecordTrackCOLMAP.cpp` — Node implementation +- `Config/RecordTrackCOLMAP.nosdef` — Node definition (pins, functions, metadata) + +### Modified files +- `Source/TrackMain.cpp` — Added `RecordTrackCOLMAP` to the `TrackNode` enum and `ExportNodeFunctions` switch +- `Track.noscfg` — Added `Config/RecordTrackCOLMAP.nosdef` to `node_definitions` + +## Node Design + +### Pins +| Pin | Type | Direction | Description | +|-----|------|-----------|-------------| +| Track | `nos.track.Track` | Input | Incoming tracking data | +| Track Out | `nos.track.Track` | Output (only) | Pass-through of input | +| Output Directory | `string` | Property | Folder picker for output | +| Image Resolution | `nos.fb.vec2u` | Property | Width/height (default 1920x1080) | +| Record | `bool` | Property | Mirrors Record/Stop functions | +| Frame Count | `uint` | Output (only) | Frames in buffer | + +### Functions +| Function | Behavior | +|----------|----------| +| Record | Validates folder is empty, clears buffer, starts recording. Orphaned while recording. | +| Stop | Stops recording (does NOT save). Orphaned while idle. | +| Save | Writes `cameras.txt` + `images.txt` to disk. Does not clear buffer. | +| Clear | Clears frame buffer and resets count. | +| Open Folder | Opens output directory in explorer (Windows) or xdg-open (Linux). | + +### State Management +- Record pin and functions are kept in sync bidirectionally. A `SyncingRecordPin` bool guard prevents re-entrant loops between pin changes and function calls. +- Function orphan states: Record/Stop toggle via `SetNodeOrphanState` using a `Name -> UUID` map built in constructor. +- Status messages show recording state + frame count, and persist error messages (e.g., "Target folder is not empty") via `LastError` until user changes the output directory. +- Non-empty folder check: Recording fails with a FAILURE status if the target folder already has files. + +### COLMAP Output Format +- `cameras.txt`: One OPENCV camera per frame — `fx, fy, cx, cy, k1, k2, p1, p2` derived from Track FOV, sensor size, pixel aspect ratio, lens distortion. +- `images.txt`: Per-frame pose — Euler angles converted to quaternion (world-to-camera), translation as `t = -R * C`. + +## Known Review Points +- Euler-to-quaternion convention: The Track's rotation fields (roll/tilt/pan) are passed through `glm::quat(eulerRadians)` then inverted for COLMAP's world-to-camera convention. May need validation against actual tracker output. +- One camera per frame: Each frame gets its own camera entry. This handles zoom/FOV changes but may be unusual for COLMAP workflows with constant intrinsics. +- No `points3D.txt`: COLMAP expects this file too (can be empty). Not currently written. +- `std::system()` for Open Folder: Works but is a simple shell call. Could be replaced with platform APIs if needed. + +## Build +``` +./nodos dev build -p Project13 --target nosTrack +``` diff --git a/Plugins/nosTrack/CMakeLists.txt b/Plugins/nosTrack/CMakeLists.txt index 26c05e2b..af2df47c 100644 --- a/Plugins/nosTrack/CMakeLists.txt +++ b/Plugins/nosTrack/CMakeLists.txt @@ -1,6 +1,6 @@ # Copyright MediaZ Teknoloji A.S. All Rights Reserved. -set(MODULE_DEPENDENCIES "nos.sys.track-1.0") +set(MODULE_DEPENDENCIES "nos.sys.track-1.1") set(dep_idx 0) foreach(module_name_version ${MODULE_DEPENDENCIES}) # module_name_version: - @@ -13,4 +13,7 @@ endforeach() list(APPEND MODULE_DEPENDENCIES_TARGETS ${NOS_PLUGIN_SDK_TARGET}) +nos_generate_flatbuffers("${CMAKE_CURRENT_SOURCE_DIR}/Config" "${CMAKE_CURRENT_SOURCE_DIR}/Source" "cpp" "" nosTrack_generated) +list(APPEND MODULE_DEPENDENCIES_TARGETS nosTrack_generated) + nos_add_plugin("nosTrack" "${MODULE_DEPENDENCIES_TARGETS}" "${CMAKE_CURRENT_LIST_DIR}/External/asio/asio/include") diff --git a/Plugins/nosTrack/Config/PlaybackMode.fbs b/Plugins/nosTrack/Config/PlaybackMode.fbs new file mode 100644 index 00000000..c0d1d952 --- /dev/null +++ b/Plugins/nosTrack/Config/PlaybackMode.fbs @@ -0,0 +1,9 @@ +namespace nos.track; + +// Selects how PlaybackTrackCOLMAP indexes into the recorded frames. +enum PlaybackTrackMode : uint +{ + FrameIndex = 0, // Use the InFrameIndex pin as a 0-based offset. + Timecode = 1, // Look up by Timecode string from timecodes.txt. + FrameNumber = 2, // Look up by FrameNumber column from timecodes.txt. +} diff --git a/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef new file mode 100644 index 00000000..7f1604f2 --- /dev/null +++ b/Plugins/nosTrack/Config/PlaybackTrackCOLMAP.nosdef @@ -0,0 +1,107 @@ +{ + "nodes": [ + { + "class_name": "PlaybackTrackCOLMAP", + "menu_info": { + "category": "nosTrack", + "display_name": "Playback Track (COLMAP)", + "name_aliases": [ "colmap", "import camera", "playback camera" ] + }, + "node": { + "class_name": "PlaybackTrackCOLMAP", + "display_name": "Playback Track (COLMAP)", + "contents_type": "Job", + "always_execute": true, + "description": "Loads camera track from COLMAP-spec cameras.txt + images.txt.\nReads world-to-camera poses in the COLMAP frame (RH, +X right, +Y down, +Z forward) and converts to the chosen TargetFrame.\nWhen an extras.txt sidecar is present, original Euler/FOV/sensor metadata is restored verbatim (no quaternion round-trip drift).", + "pins": [ + { + "name": "InputDirectory", + "display_name": "Input Directory", + "type_name": "string", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { "type": "FOLDER_PICKER" }, + "description": "Directory with cameras.txt + images.txt (and optional timecodes.txt / extras.txt sidecars)." + }, + { + "name": "TargetFrame", + "display_name": "Target Frame", + "type_name": "nos.sys.track.CoordinateFrame", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "LH_ZUp_FwdX_RightY", + "description": "Coordinate frame of the produced Track.\nCOLMAP poses are converted into this frame.\nDefault matches FreeD / UE convention." + }, + { + "name": "Mode", + "display_name": "Mode", + "type_name": "nos.track.PlaybackTrackMode", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "FrameIndex", + "description": "Selects how to index frames.\nFrameIndex uses InFrameIndex.\nTimecode / FrameNumber look up via timecodes.txt sidecar.\nThe unused index pin becomes PASSIVE." + }, + { + "name": "InFrameIndex", + "display_name": "Frame Index", + "type_name": "uint", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0, + "description": "0-based frame index. Used when Mode=FrameIndex." + }, + { + "name": "InTimecode", + "display_name": "Timecode", + "type_name": "string", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "description": "Timecode string (HH:MM:SS:FF) to look up. Used when Mode=Timecode. Requires timecodes.txt." + }, + { + "name": "InFrameNumber", + "display_name": "Frame Number", + "type_name": "uint", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0, + "description": "Absolute frame number to look up. Used when Mode=FrameNumber. Requires timecodes.txt." + }, + { + "name": "OutFrameIndex", + "display_name": "Frame Index", + "type_name": "uint", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "data": 0, + "description": "Current playback frame index." + }, + { + "name": "FrameCount", + "display_name": "Frame Count", + "type_name": "uint", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "data": 0, + "description": "Total frames loaded." + }, + { + "name": "Track", + "type_name": "nos.sys.track.Track", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "description": "Track for the current frame, expressed in the TargetFrame convention." + } + ], + "functions": [ + { + "class_name": "PlaybackTrackCOLMAP_OpenFolder", + "display_name": "Open Folder", + "contents_type": "Job", + "pins": [] + } + ] + } + } + ] +} diff --git a/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef new file mode 100644 index 00000000..c26f52cb --- /dev/null +++ b/Plugins/nosTrack/Config/RecordTrackCOLMAP.nosdef @@ -0,0 +1,120 @@ +{ + "nodes": [ + { + "class_name": "RecordTrackCOLMAP", + "menu_info": { + "category": "nosTrack", + "display_name": "Record Track (COLMAP)", + "name_aliases": [ "colmap", "export camera", "record camera" ] + }, + "node": { + "class_name": "RecordTrackCOLMAP", + "display_name": "Record Track (COLMAP)", + "contents_type": "Job", + "always_execute": true, + "description": "Records camera track data each frame while Record is true.\nOn falling edge (after MinOffFrames debounce) writes COLMAP-spec cameras.txt + images.txt and clears the buffer.\nIntrinsics come from FOV/sensor/distortion. Extrinsics are written as world-to-camera in the COLMAP frame (RH, +X right, +Y down, +Z forward).\nSet SourceFrame to match the convention of the connected Track.", + "pins": [ + { + "name": "Timecode", + "display_name": "Timecode", + "type_name": "string", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "description": "Optional SMPTE timecode (HH:MM:SS:FF). Written to timecodes.txt sidecar when non-empty." + }, + { + "name": "FrameNumber", + "display_name": "Frame Number", + "type_name": "uint", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 0, + "description": "Optional absolute frame number paired with Timecode. Written to timecodes.txt sidecar." + }, + { + "name": "OutputDirectory", + "display_name": "Output Directory", + "type_name": "string", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { "type": "FOLDER_PICKER" }, + "description": "Where cameras.txt and images.txt are written when recording stops. Must be empty to start recording." + }, + { + "name": "ImageResolution", + "display_name": "Image Resolution", + "type_name": "nos.fb.vec2u", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { + "x": 1920, + "y": 1080 + }, + "description": "Image WIDTH/HEIGHT in pixels. Used to compute focal length and principal point for cameras.txt." + }, + { + "name": "SourceFrame", + "display_name": "Source Frame", + "type_name": "nos.sys.track.CoordinateFrame", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "LH_ZUp_FwdX_RightY", + "description": "Coordinate frame of the connected Track.\nUsed to convert location and rotation into the COLMAP frame before writing.\nDefault matches FreeD / UE convention." + }, + { + "name": "Record", + "type_name": "bool", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": false, + "description": "Drives recording.\nRising edge: clear buffer and start.\nFalling edge (after MinOffFrames): stop and write files.\nFails to start if OutputDirectory is non-empty." + }, + { + "name": "MinOffFrames", + "display_name": "Min Off Frames", + "type_name": "uint", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1, + "min": "1", + "description": "Debounce: minimum consecutive Record=false frames before stopping. Default 1 = stop immediately. Use 5-15 to ride out short upstream glitches (e.g. SDI bit flips on a camera-derived flag)." + }, + { + "name": "RecordingFrame", + "display_name": "Recording Frame", + "type_name": "uint", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "data": 0, + "description": "Current recording frame index. 0 when not recording." + }, + { + "name": "FrameCount", + "display_name": "Frame Count", + "type_name": "uint", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "data": 0, + "description": "Frames in the buffer." + }, + { + "name": "InTrack", + "display_name": "Track", + "type_name": "nos.sys.track.Track", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "description": "Camera track to record. Interpreted in the SourceFrame convention (location, rotation Euler, FOV, sensor, lens distortion)." + }, + { + "name": "OutTrack", + "display_name": "Track", + "type_name": "nos.sys.track.Track", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "description": "Pass-through of InTrack." + } + ] + } + } + ] +} diff --git a/Plugins/nosTrack/Config/TrackTransform.nosdef b/Plugins/nosTrack/Config/TrackTransform.nosdef new file mode 100644 index 00000000..cb198734 --- /dev/null +++ b/Plugins/nosTrack/Config/TrackTransform.nosdef @@ -0,0 +1,54 @@ +{ + "nodes": [ + { + "class_name": "TrackTransform", + "menu_info": { + "category": "Track|Coordinate System", + "display_name": "Track Transform" + }, + "node": { + "class_name": "TrackTransform", + "contents_type": "Job", + "description": "Transforms a Track between coordinate frames.\nThe Source and Target enums select axis assignments, handedness, and the Euler convention used for the rotation field.\nLocation: basis-changed (Source -> Target), then multiplied by WorldScale (e.g. 0.01 for cm -> m, 100 for m -> cm).\nRotation: built in the source Euler convention, conjugated by the basis-change matrix, re-extracted in the target convention.\nOther Track fields (fov, focus, sensor_size, lens_distortion, ...) pass through unchanged.", + "pins": [ + { + "name": "In", + "type_name": "nos.sys.track.Track", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Source", + "type_name": "nos.sys.track.CoordinateFrame", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "LH_ZUp_FwdX_RightY", + "description": "Coordinate system convention of the input Track." + }, + { + "name": "Target", + "type_name": "nos.sys.track.CoordinateFrame", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "RH_YUp_FwdNegZ_RightX", + "description": "Coordinate system convention of the output Track." + }, + { + "name": "WorldScale", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1.0, + "description": "Uniform scale applied only to the output location after the basis change. Use to convert linear units (e.g. 0.01 for cm -> m, 100 for m -> cm). Does not affect rotation, fov, sensor size, focus, or lens distortion." + }, + { + "name": "Out", + "type_name": "nos.sys.track.Track", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_OR_PROPERTY" + } + ] + } + } + ] +} diff --git a/Plugins/nosTrack/Source/CoordinateFrameConv.h b/Plugins/nosTrack/Source/CoordinateFrameConv.h new file mode 100644 index 00000000..777fea34 --- /dev/null +++ b/Plugins/nosTrack/Source/CoordinateFrameConv.h @@ -0,0 +1,100 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. +// Frame-conversion helpers shared by TrackTransform / RecordTrackCOLMAP / +// PlaybackTrackCOLMAP. Encodes per-frame Euler conventions and basis-change +// matrices to the COLMAP camera/world frame. +#pragma once + +#include "nosSysTrack/Track_generated.h" +#include +#include + +namespace nos::track::convention +{ + +using Frame = sys::track::CoordinateFrame; + +// Basis matrix S for a CoordinateFrame: maps semantic (forward, right, up) +// to engine coords (vx, vy, vz). v_engine = S * (forward, right, up). +// det(S) > 0 for left-handed frames, < 0 for right-handed (with this ordering). +inline glm::dmat3 BasisMatrix(Frame frame) +{ + switch (frame) + { + case Frame::LH_ZUp_FwdX_RightY: + // vx = forward, vy = right, vz = up. + return glm::dmat3(1.0); + case Frame::RH_YUp_FwdNegZ_RightX: + // vx = right, vy = up, vz = -forward. + return glm::dmat3( + glm::dvec3( 0.0, 0.0, -1.0), // M * (1,0,0) = forward column + glm::dvec3( 1.0, 0.0, 0.0), // M * (0,1,0) = right column + glm::dvec3( 0.0, 1.0, 0.0)); // M * (0,0,1) = up column + } + return glm::dmat3(1.0); +} + +// COLMAP camera/world frame: X right, Y down, Z forward (RH). +// Provided as a basis matrix in the same (forward, right, up) convention so +// it can be combined with BasisMatrix to build cross-frame conversions. +inline glm::dmat3 ColmapBasisMatrix() +{ + return glm::dmat3( + glm::dvec3( 0.0, 0.0, 1.0), // forward -> +Z + glm::dvec3( 1.0, 0.0, 0.0), // right -> +X + glm::dvec3( 0.0, -1.0, 0.0)); // up -> -Y (Y is down) +} + +// Build R_c2w in `frame` from Track.rotation Euler degrees. +inline glm::dmat3 EulerToMat(Frame frame, glm::dvec3 const& degRot) +{ + glm::dvec3 r = glm::radians(degRot); + switch (frame) + { + case Frame::LH_ZUp_FwdX_RightY: + // FRotator: rot.x = roll, rot.y = pitch, rot.z = yaw, intrinsic ZYX. + // UE sign convention has +pitch = look up and +roll = bank right via + // LH-rule rotations, equivalent to standard-RH Rz(yaw) * Ry(-pitch) * Rx(-roll). + return glm::dmat3(glm::eulerAngleZYX(r.z, -r.y, -r.x)); + case Frame::RH_YUp_FwdNegZ_RightX: + // rot.x = pitch, rot.y = yaw, rot.z = roll, intrinsic YXZ: + // R = Ry(yaw) * Rx(pitch) * Rz(roll), all standard-RH formulas. + return glm::dmat3(glm::eulerAngleYXZ(r.y, r.x, r.z)); + } + return glm::dmat3(1.0); +} + +// Inverse of EulerToMat: extract Euler degrees in `frame`'s convention. +// Output is packed into the (rot.x, rot.y, rot.z) Track layout for that frame. +inline glm::dvec3 MatToEuler(Frame frame, glm::dmat3 const& R) +{ + glm::dmat4 M(R); + double a = 0.0, b = 0.0, c = 0.0; + switch (frame) + { + case Frame::LH_ZUp_FwdX_RightY: + glm::extractEulerAngleZYX(M, a, b, c); // a=yaw, b=pitch, c=roll + // Negate pitch and roll back to UE sign convention; pack as (roll, pitch, yaw). + return glm::degrees(glm::dvec3(-c, -b, a)); + case Frame::RH_YUp_FwdNegZ_RightX: + glm::extractEulerAngleYXZ(M, a, b, c); // a=yaw, b=pitch, c=roll + // Pack as (pitch, yaw, roll). + return glm::degrees(glm::dvec3(b, a, c)); + } + return glm::dvec3(0.0); +} + +// Basis-change M from `frame` to COLMAP frame: M = S_colmap * S_frame^-1. +// For a vector: v_colmap = M * v_frame. +// For a rotation matrix: R_colmap = M * R_frame * M^-1. +inline glm::dmat3 BasisChangeToColmap(Frame frame) +{ + return ColmapBasisMatrix() * glm::inverse(BasisMatrix(frame)); +} + +// Inverse of BasisChangeToColmap. +inline glm::dmat3 BasisChangeFromColmap(Frame frame) +{ + return BasisMatrix(frame) * glm::inverse(ColmapBasisMatrix()); +} + +} // namespace nos::track::convention diff --git a/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp new file mode 100644 index 00000000..56271a01 --- /dev/null +++ b/Plugins/nosTrack/Source/PlaybackTrackCOLMAP.cpp @@ -0,0 +1,568 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include "nosSysTrack/Track_generated.h" +#include "PlaybackMode_generated.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "CoordinateFrameConv.h" + +namespace nos::track +{ + +NOS_REGISTER_NAME_SPACED(Playback_InputDirectory, "InputDirectory"); +NOS_REGISTER_NAME_SPACED(Playback_TargetFrame, "TargetFrame"); +NOS_REGISTER_NAME_SPACED(Playback_Mode, "Mode"); +NOS_REGISTER_NAME_SPACED(Playback_InFrameIndex, "InFrameIndex"); +NOS_REGISTER_NAME_SPACED(Playback_InTimecode, "InTimecode"); +NOS_REGISTER_NAME_SPACED(Playback_InFrameNumber, "InFrameNumber"); +NOS_REGISTER_NAME_SPACED(Playback_OutFrameIndex, "OutFrameIndex"); +NOS_REGISTER_NAME_SPACED(Playback_FrameCount, "FrameCount"); + +NOS_REGISTER_NAME(PlaybackTrackCOLMAP_OpenFolder); + +struct COLMAPCamera +{ + uint32_t Id = 0; + std::string Model; + uint32_t Width = 0; + uint32_t Height = 0; + float Fx = 0, Fy = 0, Cx = 0, Cy = 0; + float K1 = 0, K2 = 0, P1 = 0, P2 = 0; +}; + +struct COLMAPImage +{ + uint32_t Id = 0; + glm::quat Q{1, 0, 0, 0}; // R_w2c in COLMAP camera frame. + glm::vec3 T{0}; // t = -R_w2c * camera_world_position (COLMAP world frame). + uint32_t CameraId = 0; +}; + +struct TimecodeEntry +{ + std::string Timecode; + uint32_t FrameNumber = 0; +}; + +struct ExtrasEntry +{ + bool Present = false; + float Zoom = 0; + float Focus = 0; + float FocusDistance = 0; + float RenderRatio = 0; + float NodalOffset = 0; + float DistortionScale = 0; + float SensorWmm = 0; + float SensorHmm = 0; + float RotX = 0; + float RotY = 0; + float RotZ = 0; +}; + +struct PlaybackTrackCOLMAPContext : NodeContext +{ + std::string InputDir; + convention::Frame TargetFrame = convention::Frame::LH_ZUp_FwdX_RightY; + PlaybackTrackMode Mode = PlaybackTrackMode::FrameIndex; + uint32_t FrameIndex = 0; + std::string InTimecode; + uint32_t InFrameNumber = 0; + std::string LastError; + std::vector Frames; + std::vector Timecodes; // empty or same size as Frames + std::unordered_map TimecodeToIndex; + std::unordered_map FrameNumberToIndex; + uint32_t CurrentFrame = 0; + + PlaybackTrackCOLMAPContext(nosFbNodePtr node) : NodeContext(node) + { + if (node->pins()) + { + for (auto* pin : *node->pins()) + { + auto name = nos::Name(pin->name()->c_str()); + if (flatbuffers::IsFieldPresent(pin, fb::Pin::VT_DATA)) + { + nosBuffer value = {.Data = (void*)pin->data()->data(), .Size = pin->data()->size()}; + OnPinValueChanged(name, *pin->id(), value); + } + } + } + ApplyModeOrphanStates(); + UpdateStatus(); + } + + void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer val) override + { + if (pinName == NSN_Playback_InputDirectory) + { + InputDir = InterpretPinValue(val.Data); + LastError.clear(); + if (!InputDir.empty()) + LoadFromDirectory(); + else + UpdateStatus(); + } + else if (pinName == NSN_Playback_TargetFrame) + { + TargetFrame = *(convention::Frame*)val.Data; + if (!InputDir.empty()) + LoadFromDirectory(); + } + else if (pinName == NSN_Playback_Mode) + { + Mode = *(PlaybackTrackMode*)val.Data; + ApplyModeOrphanStates(); + } + else if (pinName == NSN_Playback_InFrameIndex) + FrameIndex = *(uint32_t*)val.Data; + else if (pinName == NSN_Playback_InTimecode) + InTimecode = InterpretPinValue(val.Data); + else if (pinName == NSN_Playback_InFrameNumber) + InFrameNumber = *(uint32_t*)val.Data; + } + + void ApplyModeOrphanStates() + { + auto state = [](bool active) { + return active ? fb::PinOrphanStateType::ACTIVE : fb::PinOrphanStateType::PASSIVE; + }; + const bool useIdx = Mode == PlaybackTrackMode::FrameIndex; + const bool useTC = Mode == PlaybackTrackMode::Timecode; + const bool useFN = Mode == PlaybackTrackMode::FrameNumber; + SetPinOrphanState(NSN_Playback_InFrameIndex, state(useIdx)); + SetPinOrphanState(NSN_Playback_InTimecode, state(useTC)); + SetPinOrphanState(NSN_Playback_InFrameNumber, state(useFN)); + } + + void UpdateFrameCountPin() + { + uint32_t count = (uint32_t)Frames.size(); + SetPinValue(NSN_Playback_FrameCount, nosBuffer{.Data = &count, .Size = sizeof(count)}); + } + + void UpdateFrameIndexPin() + { + SetPinValue(NSN_Playback_OutFrameIndex, nosBuffer{.Data = &CurrentFrame, .Size = sizeof(CurrentFrame)}); + } + + void UpdateStatus() + { + if (!LastError.empty()) + SetNodeStatusMessage(LastError, fb::NodeStatusMessageType::FAILURE); + else if (InputDir.empty()) + SetNodeStatusMessage("Set input directory", fb::NodeStatusMessageType::WARNING); + else if (Frames.empty()) + SetNodeStatusMessage("No data loaded", fb::NodeStatusMessageType::WARNING); + else + SetNodeStatusMessage("Loaded (" + std::to_string(Frames.size()) + " frames)", fb::NodeStatusMessageType::INFO); + } + + // --- Parsing --- + + bool LoadFromDirectory() + { + if (InputDir.empty()) + { + LastError = "Set input directory"; + UpdateStatus(); + return false; + } + + std::filesystem::path dir = nos::Utf8ToPath(InputDir); + auto camerasPath = dir / "cameras.txt"; + auto imagesPath = dir / "images.txt"; + + if (!std::filesystem::exists(camerasPath)) + { + LastError = "cameras.txt not found"; + UpdateStatus(); + return false; + } + if (!std::filesystem::exists(imagesPath)) + { + LastError = "images.txt not found"; + UpdateStatus(); + return false; + } + + std::unordered_map cameras; + if (!ParseCamerasTxt(camerasPath, cameras)) + return false; + + std::vector images; + if (!ParseImagesTxt(imagesPath, images)) + return false; + + if (images.empty()) + { + LastError = "No images found in images.txt"; + UpdateStatus(); + return false; + } + + Frames.clear(); + Frames.reserve(images.size()); + Timecodes.clear(); + + auto timecodesPath = dir / "timecodes.txt"; + if (std::filesystem::exists(timecodesPath)) + ParseTimecodesTxt(timecodesPath, images.size()); + + std::vector extras; + auto extrasPath = dir / "extras.txt"; + if (std::filesystem::exists(extrasPath)) + ParseExtrasTxt(extrasPath, images.size(), extras); + + // Inverse of RecordTrackCOLMAP::WriteImagesTxt: + // images.txt holds R_w2c in COLMAP frame, t = -R_w2c * pos_colmap. + // pos_colmap = -R_c2w_colmap * t (R_c2w_colmap = R_w2c^T) + // pos_target = M^-1 * pos_colmap + // R_c2w_target = M^-1 * R_c2w_colmap * M + // Track.rotation = MatToEuler(TargetFrame, R_c2w_target) + const glm::dmat3 Minv = convention::BasisChangeFromColmap(TargetFrame); + const glm::dmat3 M = glm::inverse(Minv); + + for (size_t i = 0; i < images.size(); ++i) + { + auto& img = images[i]; + sys::track::TTrack trackData{}; + auto camIt = cameras.find(img.CameraId); + const ExtrasEntry* ex = (i < extras.size() && extras[i].Present) ? &extras[i] : nullptr; + + glm::dmat3 R_w2c = glm::dmat3(glm::mat3_cast(img.Q)); + glm::dmat3 R_c2w_colmap = glm::transpose(R_w2c); + glm::dvec3 pos_colmap = -R_c2w_colmap * glm::dvec3(img.T); + + glm::dvec3 pos_target = Minv * pos_colmap; + glm::vec3 locF((float)pos_target.x, (float)pos_target.y, (float)pos_target.z); + trackData.location = reinterpret_cast(locF); + + // Rotation: prefer the original Euler from extras (avoids quaternion- + // to-Euler ambiguity near gimbal lock); fall back to extracting from + // the COLMAP rotation matrix when no extras sidecar exists. + if (ex) + { + glm::vec3 euler(ex->RotX, ex->RotY, ex->RotZ); + trackData.rotation = reinterpret_cast(euler); + } + else + { + glm::dmat3 R_c2w_target = Minv * R_c2w_colmap * M; + glm::dvec3 eulerD = convention::MatToEuler(TargetFrame, R_c2w_target); + glm::vec3 eulerF((float)eulerD.x, (float)eulerD.y, (float)eulerD.z); + trackData.rotation = reinterpret_cast(eulerF); + } + + if (camIt != cameras.end()) + { + auto& cam = camIt->second; + if (cam.Fx > 0) + trackData.fov = glm::degrees(2.0f * std::atan(cam.Width * 0.5f / cam.Fx)); + if (cam.Fx > 0 && cam.Fy > 0) + trackData.pixel_aspect_ratio = cam.Fx / cam.Fy; + trackData.lens_distortion.mutable_k1k2() = nos::fb::vec2(cam.K1, cam.K2); + + // sensor_size: COLMAP only stores pixel dims, but Track expects mm. + // Use the extras value when present; otherwise fall back to pixels + // (matches pre-extras behaviour). + glm::vec2 sensorMm(0); + if (ex && ex->SensorWmm > 0 && ex->SensorHmm > 0) + { + sensorMm = {ex->SensorWmm, ex->SensorHmm}; + trackData.sensor_size = nos::fb::vec2(sensorMm.x, sensorMm.y); + } + else + { + trackData.sensor_size = nos::fb::vec2(cam.Width, cam.Height); + } + + // center_shift: invert the (cx, cy) encoding written by record. + // Needs sensor_size in mm to be meaningful, so only reconstructed + // when extras provided it. + if (sensorMm.x > 0 && cam.Width > 0 && sensorMm.y > 0 && cam.Height > 0) + { + glm::vec2 shift{ + (cam.Cx - cam.Width * 0.5f) * sensorMm.x / cam.Width, + (cam.Cy - cam.Height * 0.5f) * sensorMm.y / cam.Height}; + trackData.lens_distortion.mutable_center_shift() = reinterpret_cast(shift); + } + } + + if (ex) + { + trackData.zoom = ex->Zoom; + trackData.focus = ex->Focus; + trackData.focus_distance = ex->FocusDistance; + trackData.render_ratio = ex->RenderRatio; + trackData.nodal_offset = ex->NodalOffset; + trackData.lens_distortion.mutate_distortion_scale(ex->DistortionScale); + } + + Frames.push_back(std::move(trackData)); + } + + CurrentFrame = 0; + LastError.clear(); + UpdateFrameCountPin(); + UpdateFrameIndexPin(); + UpdateStatus(); + nosEngine.LogI("PlaybackTrackCOLMAP: Loaded %zu frames from %s", Frames.size(), InputDir.c_str()); + return true; + } + + bool ParseCamerasTxt(const std::filesystem::path& path, std::unordered_map& cameras) + { + std::ifstream file(path); + if (!file.is_open()) + { + LastError = "Cannot open cameras.txt"; + UpdateStatus(); + return false; + } + + std::string line; + while (std::getline(file, line)) + { + if (line.empty() || line[0] == '#') + continue; + std::istringstream ss(line); + COLMAPCamera cam; + ss >> cam.Id >> cam.Model >> cam.Width >> cam.Height; + if (cam.Model == "OPENCV") + ss >> cam.Fx >> cam.Fy >> cam.Cx >> cam.Cy >> cam.K1 >> cam.K2 >> cam.P1 >> cam.P2; + else if (cam.Model == "PINHOLE") + ss >> cam.Fx >> cam.Fy >> cam.Cx >> cam.Cy; + else if (cam.Model == "SIMPLE_PINHOLE") + { + float f; + ss >> f >> cam.Cx >> cam.Cy; + cam.Fx = cam.Fy = f; + } + else if (cam.Model == "SIMPLE_RADIAL") + { + float f; + ss >> f >> cam.Cx >> cam.Cy >> cam.K1; + cam.Fx = cam.Fy = f; + } + else if (cam.Model == "RADIAL") + { + float f; + ss >> f >> cam.Cx >> cam.Cy >> cam.K1 >> cam.K2; + cam.Fx = cam.Fy = f; + } + else + { + nosEngine.LogW("PlaybackTrackCOLMAP: Unsupported camera model '%s', treating as PINHOLE", cam.Model.c_str()); + ss >> cam.Fx >> cam.Fy >> cam.Cx >> cam.Cy; + } + cameras[cam.Id] = cam; + } + return true; + } + + bool ParseImagesTxt(const std::filesystem::path& path, std::vector& images) + { + std::ifstream file(path); + if (!file.is_open()) + { + LastError = "Cannot open images.txt"; + UpdateStatus(); + return false; + } + + std::string line; + while (std::getline(file, line)) + { + if (line.empty() || line[0] == '#') + continue; + std::istringstream ss(line); + COLMAPImage img; + float qw, qx, qy, qz; + std::string name; + ss >> img.Id >> qw >> qx >> qy >> qz + >> img.T.x >> img.T.y >> img.T.z + >> img.CameraId >> name; + img.Q = glm::quat(qw, qx, qy, qz); + images.push_back(img); + // Skip POINTS2D line + std::getline(file, line); + } + + std::sort(images.begin(), images.end(), [](auto& a, auto& b) { return a.Id < b.Id; }); + return true; + } + + void ParseExtrasTxt(const std::filesystem::path& path, size_t expectedCount, std::vector& outExtras) + { + std::ifstream file(path); + if (!file.is_open()) + return; + std::unordered_map byId; + std::string line; + while (std::getline(file, line)) + { + if (line.empty() || line[0] == '#') + continue; + std::istringstream ss(line); + uint32_t id = 0; + ExtrasEntry e; + ss >> id >> e.Zoom >> e.Focus >> e.FocusDistance >> e.RenderRatio + >> e.NodalOffset >> e.DistortionScale + >> e.SensorWmm >> e.SensorHmm + >> e.RotX >> e.RotY >> e.RotZ; + if (!ss.fail()) + { + e.Present = true; + byId[id] = e; + } + } + outExtras.assign(expectedCount, ExtrasEntry{}); + for (size_t i = 0; i < expectedCount; ++i) + { + auto it = byId.find(uint32_t(i + 1)); + if (it != byId.end()) + outExtras[i] = it->second; + } + } + + void ParseTimecodesTxt(const std::filesystem::path& path, size_t expectedCount) + { + std::ifstream file(path); + if (!file.is_open()) + return; + std::unordered_map byId; + std::string line; + while (std::getline(file, line)) + { + if (line.empty() || line[0] == '#') + continue; + std::istringstream ss(line); + uint32_t id = 0; + TimecodeEntry e; + ss >> id >> e.Timecode >> e.FrameNumber; + if (e.Timecode == "-") + e.Timecode.clear(); + byId[id] = std::move(e); + } + Timecodes.assign(expectedCount, TimecodeEntry{}); + TimecodeToIndex.clear(); + FrameNumberToIndex.clear(); + for (size_t i = 0; i < expectedCount; ++i) + { + auto it = byId.find(uint32_t(i + 1)); + if (it == byId.end()) + continue; + Timecodes[i] = it->second; + if (!Timecodes[i].Timecode.empty()) + TimecodeToIndex.emplace(Timecodes[i].Timecode, uint32_t(i)); + FrameNumberToIndex.emplace(Timecodes[i].FrameNumber, uint32_t(i)); + } + } + + // --- Execution --- + + bool ResolveFrameIndex(uint32_t& outIdx) + { + switch (Mode) + { + case PlaybackTrackMode::Timecode: + { + auto it = TimecodeToIndex.find(InTimecode); + if (it == TimecodeToIndex.end()) + return false; + outIdx = it->second; + return true; + } + case PlaybackTrackMode::FrameNumber: + { + auto it = FrameNumberToIndex.find(InFrameNumber); + if (it == FrameNumberToIndex.end()) + return false; + outIdx = it->second; + return true; + } + case PlaybackTrackMode::FrameIndex: + default: + outIdx = FrameIndex < (uint32_t)Frames.size() ? FrameIndex : (uint32_t)Frames.size() - 1; + return true; + } + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + if (Frames.empty()) + { + sys::track::TTrack empty{}; + auto buf = nos::Buffer::From(empty); + SetPinValue(NOS_NAME("Track"), {.Data = buf.Data(), .Size = buf.Size()}); + return NOS_RESULT_SUCCESS; + } + + uint32_t frameIdx = 0; + if (!ResolveFrameIndex(frameIdx)) + frameIdx = CurrentFrame < (uint32_t)Frames.size() ? CurrentFrame : 0; + CurrentFrame = frameIdx; + + auto buf = nos::Buffer::From(Frames[frameIdx]); + SetPinValue(NOS_NAME("Track"), {.Data = buf.Data(), .Size = buf.Size()}); + UpdateFrameIndexPin(); + + return NOS_RESULT_SUCCESS; + } + + static nosResult GetFunctions(size_t* count, nosName* names, nosPfnNodeFunctionExecute* fns) + { + *count = 1; + if (!names || !fns) + return NOS_RESULT_SUCCESS; + + names[0] = NOS_NAME_STATIC("PlaybackTrackCOLMAP_OpenFolder"); + fns[0] = [](void* ctx, nosFunctionExecuteParams*) { + auto* self = static_cast(ctx); + if (self->InputDir.empty()) + { + nosEngine.LogW("PlaybackTrackCOLMAP: Input directory not set"); + return NOS_RESULT_FAILED; + } + std::filesystem::path dir = nos::Utf8ToPath(self->InputDir); + if (!std::filesystem::exists(dir)) + { + nosEngine.LogW("PlaybackTrackCOLMAP: Directory does not exist: %s", self->InputDir.c_str()); + return NOS_RESULT_FAILED; + } + // TODO: Replace std::system with platform APIs (ShellExecuteW / posix_spawnp) +#if defined(_WIN32) + std::string cmd = "explorer \"" + nos::PathToUtf8(dir) + "\""; +#elif defined(__APPLE__) + std::string cmd = "open \"" + nos::PathToUtf8(dir) + "\""; +#else + std::string cmd = "xdg-open \"" + nos::PathToUtf8(dir) + "\""; +#endif + std::system(cmd.c_str()); + return NOS_RESULT_SUCCESS; + }; + + return NOS_RESULT_SUCCESS; + } +}; + +void RegisterPlaybackTrackCOLMAP(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("PlaybackTrackCOLMAP"), PlaybackTrackCOLMAPContext, fn); +} + +} // namespace nos::track diff --git a/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp new file mode 100644 index 00000000..661d06e3 --- /dev/null +++ b/Plugins/nosTrack/Source/RecordTrackCOLMAP.cpp @@ -0,0 +1,463 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include "nosSysTrack/Track_generated.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "CoordinateFrameConv.h" + +namespace nos::track +{ + +NOS_REGISTER_NAME(OutputDirectory); +NOS_REGISTER_NAME(ImageResolution); +NOS_REGISTER_NAME(SourceFrame); +NOS_REGISTER_NAME(Record); +NOS_REGISTER_NAME(MinOffFrames); +NOS_REGISTER_NAME(FrameCount); +NOS_REGISTER_NAME(RecordingFrame); + +struct RecordedFrame +{ + glm::vec3 Location; + glm::vec3 Rotation; // Euler degrees in the SourceFrame's convention. + float FOV; + float Zoom; + float Focus; + float RenderRatio; + glm::vec2 SensorSize; + float PixelAspectRatio; + float NodalOffset; + float FocusDistance; + float K1; + float K2; + glm::vec2 CenterShift; + float DistortionScale; + std::string Timecode; + uint32_t FrameNumber; +}; + +struct RecordTrackCOLMAPContext : NodeContext +{ + std::string OutputDir; + nosVec2u ImageResolution = {1920, 1080}; + convention::Frame SourceFrame = convention::Frame::LH_ZUp_FwdX_RightY; + bool Recording = false; + uint32_t ConsecutiveOffFrames = 0; + bool LastRequestRecord = false; + std::string LastError; + std::vector Frames; + nosVec2u DeltaSeconds{}; // {numerator, denominator}; 0/0 if not in fixed-step mode + + RecordTrackCOLMAPContext(nosFbNodePtr node) : NodeContext(node) + { + if (node->pins()) + { + for (auto* pin : *node->pins()) + { + auto name = nos::Name(pin->name()->c_str()); + if (flatbuffers::IsFieldPresent(pin, fb::Pin::VT_DATA)) + { + nosBuffer value = {.Data = (void*)pin->data()->data(), .Size = pin->data()->size()}; + OnPinValueChanged(name, *pin->id(), value); + } + } + } + UpdateStatus(); + } + + bool StartRecording() + { + std::string error; + if (!CanStartRecording(error)) + { + LastError = std::move(error); + UpdateStatus(); + return false; + } + LastError.clear(); + Frames.clear(); + Recording = true; + ConsecutiveOffFrames = 0; + UpdateFrameCountPin(); + UpdateRecordingFramePin(); + UpdateStatus(); + nosEngine.LogI("RecordTrackCOLMAP: Recording started"); + return true; + } + + void StopRecording() + { + Recording = false; + nosEngine.LogI("RecordTrackCOLMAP: Recording stopped (%zu frames in buffer)", Frames.size()); + if (!Frames.empty()) + WriteFiles(); + Frames.clear(); + UpdateFrameCountPin(); + UpdateRecordingFramePin(); + UpdateStatus(); + } + + void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer val) override + { + if (pinName == NSN_OutputDirectory) + { + OutputDir = InterpretPinValue(val.Data); + LastError.clear(); + UpdateStatus(); + } + else if (pinName == NSN_ImageResolution) + ImageResolution = *(nosVec2u*)val.Data; + else if (pinName == NSN_SourceFrame) + SourceFrame = *(convention::Frame*)val.Data; + } + + bool CanStartRecording(std::string& outError) + { + if (OutputDir.empty()) + { + outError = "Set output directory"; + return false; + } + + std::filesystem::path outDir = nos::Utf8ToPath(OutputDir); + try + { + if (std::filesystem::exists(outDir) && !std::filesystem::is_empty(outDir)) + { + outError = "Target folder is not empty"; + return false; + } + } + catch (std::filesystem::filesystem_error& e) + { + nosEngine.LogE("RecordTrackCOLMAP: %s", e.what()); + outError = e.what(); + return false; + } + return true; + } + + void UpdateFrameCountPin() + { + uint32_t count = (uint32_t)Frames.size(); + SetPinValue(NSN_FrameCount, nosBuffer{.Data = &count, .Size = sizeof(count)}); + } + + void UpdateRecordingFramePin() + { + uint32_t frame = Recording ? (uint32_t)Frames.size() : 0; + SetPinValue(NSN_RecordingFrame, nosBuffer{.Data = &frame, .Size = sizeof(frame)}); + } + + void UpdateStatus() + { + if (!LastError.empty()) + SetNodeStatusMessage(LastError, fb::NodeStatusMessageType::FAILURE); + else if (OutputDir.empty()) + SetNodeStatusMessage("Set output directory", fb::NodeStatusMessageType::WARNING); + else if (Recording) + SetNodeStatusMessage("Recording (" + std::to_string(Frames.size()) + " frames)", fb::NodeStatusMessageType::INFO); + else + SetNodeStatusMessage("Idle", fb::NodeStatusMessageType::INFO); + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + nos::NodeExecuteParams execParams(params); + + if (params->TimingInfo.TimingMode == NOS_EXECUTION_TIMING_MODE_FIXED_STEP) + DeltaSeconds = params->TimingInfo.FixedStepTiming.DeltaSeconds; + + // Pass through Track input to output + nosBuffer trackBuf{}; + for (size_t i = 0; i < params->PinCount; ++i) + { + if (params->Pins[i].Name == NOS_NAME("InTrack")) + { + trackBuf = {.Data = (void*)params->Pins[i].Data->Data, .Size = params->Pins[i].Data->Size}; + break; + } + } + SetPinValue(NOS_NAME("OutTrack"), trackBuf); + + // Drive recording state from the Record pin, with off-state debouncing to + // ride out brief glitches in the upstream signal (e.g. SDI bit flips on a + // camera-derived recording flag). Start happens immediately on a rising + // edge; stop only after MinOffFrames consecutive false frames. + const bool requestRecord = *execParams.GetPinData(NSN_Record); + const uint32_t minOffFrames = *execParams.GetPinData(NSN_MinOffFrames); + + const bool risingEdge = requestRecord && !LastRequestRecord; + LastRequestRecord = requestRecord; + + if (risingEdge && !Recording) + StartRecording(); + + if (Recording) + { + if (requestRecord) + ConsecutiveOffFrames = 0; + else if (++ConsecutiveOffFrames >= std::max(1u, minOffFrames)) + StopRecording(); + } + + if (!Recording) + return NOS_RESULT_SUCCESS; + + auto* trackData = flatbuffers::GetRoot(trackBuf.Data); + if (!trackData) + return NOS_RESULT_SUCCESS; + + RecordedFrame frame{}; + if (const char* tc = execParams.GetPinData(NOS_NAME_STATIC("Timecode"))) + frame.Timecode = tc; + frame.FrameNumber = *execParams.GetPinData(NOS_NAME_STATIC("FrameNumber")); + if (auto* loc = trackData->location()) + frame.Location = {loc->x(), loc->y(), loc->z()}; + if (auto* rot = trackData->rotation()) + frame.Rotation = {rot->x(), rot->y(), rot->z()}; + frame.FOV = trackData->fov(); + frame.Zoom = trackData->zoom(); + frame.Focus = trackData->focus(); + frame.RenderRatio = trackData->render_ratio(); + if (auto* ss = trackData->sensor_size()) + frame.SensorSize = {ss->x(), ss->y()}; + frame.PixelAspectRatio = trackData->pixel_aspect_ratio(); + frame.NodalOffset = trackData->nodal_offset(); + frame.FocusDistance = trackData->focus_distance(); + if (auto* ld = trackData->lens_distortion()) + { + frame.K1 = ld->k1k2().x(); + frame.K2 = ld->k1k2().y(); + frame.CenterShift = {ld->center_shift().x(), ld->center_shift().y()}; + frame.DistortionScale = ld->distortion_scale(); + } + Frames.push_back(frame); + + UpdateFrameCountPin(); + UpdateRecordingFramePin(); + UpdateStatus(); + + return NOS_RESULT_SUCCESS; + } + + void WriteFiles() + { + if (OutputDir.empty()) + { + nosEngine.LogE("RecordTrackCOLMAP: Output directory is empty"); + return; + } + if (Frames.empty()) + { + nosEngine.LogW("RecordTrackCOLMAP: No frames recorded"); + return; + } + + std::filesystem::path outDir = nos::Utf8ToPath(OutputDir); + try + { + if (!std::filesystem::exists(outDir)) + std::filesystem::create_directories(outDir); + } + catch (std::filesystem::filesystem_error& e) + { + nosEngine.LogE("RecordTrackCOLMAP: %s", e.what()); + return; + } + + WriteCamerasTxt(outDir); + WriteImagesTxt(outDir); + WriteTimecodesTxt(outDir); + WriteExtrasTxt(outDir); + nosEngine.LogI("RecordTrackCOLMAP: Saved %zu frames to %s", Frames.size(), OutputDir.c_str()); + } + + void WriteExtrasTxt(const std::filesystem::path& outDir) + { + // Sidecar for Track fields that don't fit COLMAP's standard cameras.txt / + // images.txt format. Keyed by IMAGE_ID so it pairs 1:1 with images.txt. + auto path = outDir / "extras.txt"; + std::ofstream file(path); + if (!file.is_open()) + { + nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); + return; + } + const char* frameName = + SourceFrame == convention::Frame::LH_ZUp_FwdX_RightY ? "LH_ZUp_FwdX_RightY" + : SourceFrame == convention::Frame::RH_YUp_FwdNegZ_RightX ? "RH_YUp_FwdNegZ_RightX" + : "Unknown"; + file << std::setprecision(12); + file << "# Nodos Track sidecar paired with images.txt by IMAGE_ID.\n"; + file << "# Carries fields that don't fit COLMAP's cameras.txt/images.txt:\n"; + file << "# - sensor_size in mm (cameras.txt only stores pixel WIDTH/HEIGHT)\n"; + file << "# - original Euler rotation in degrees (avoids quaternion round-trip drift)\n"; + file << "# - nodos-only fields with no COLMAP equivalent\n"; + file << "# SourceFrame: " << frameName << " (Euler convention used for ROT_X, ROT_Y, ROT_Z below).\n"; + file << "# IMAGE_ID, ZOOM, FOCUS, FOCUS_DISTANCE, RENDER_RATIO, NODAL_OFFSET, DISTORTION_SCALE, SENSOR_W_MM, SENSOR_H_MM, ROT_X, ROT_Y, ROT_Z\n"; + file << "# Number of entries: " << Frames.size() << "\n"; + for (size_t i = 0; i < Frames.size(); ++i) + { + const auto& f = Frames[i]; + file << (i + 1) << " " + << f.Zoom << " " + << f.Focus << " " + << f.FocusDistance << " " + << f.RenderRatio << " " + << f.NodalOffset << " " + << f.DistortionScale << " " + << f.SensorSize.x << " " << f.SensorSize.y << " " + << f.Rotation.x << " " << f.Rotation.y << " " << f.Rotation.z << "\n"; + } + } + + void WriteTimecodesTxt(const std::filesystem::path& outDir) + { + // Skip the sidecar entirely if no frame carried a timecode -- keeps the + // output minimal when the upstream graph isn't producing TC. + bool any = false; + for (auto& f : Frames) + if (!f.Timecode.empty() || f.FrameNumber != 0) { any = true; break; } + if (!any) + return; + + auto path = outDir / "timecodes.txt"; + std::ofstream file(path); + if (!file.is_open()) + { + nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); + return; + } + double dt = (DeltaSeconds.y != 0) ? (double)DeltaSeconds.x / (double)DeltaSeconds.y : 0.0; + file << "# Timecode sidecar paired with images.txt by IMAGE_ID.\n"; + file << "# First non-comment line: per-frame delta seconds (0 if recording wasn't in fixed-step timing).\n"; + file << "# IMAGE_ID, TIMECODE, FRAME_NUMBER\n"; + file << "# Number of entries: " << Frames.size() << "\n"; + file << std::setprecision(12) << dt << "\n"; + for (size_t i = 0; i < Frames.size(); ++i) + { + const auto& f = Frames[i]; + file << (i + 1) << " " + << (f.Timecode.empty() ? "-" : f.Timecode) << " " + << f.FrameNumber << "\n"; + } + } + + float ComputeFocalLengthPixels(const RecordedFrame& frame) const + { + if (frame.FOV <= 0.0f) + return static_cast(ImageResolution.x); + float fovRad = glm::radians(frame.FOV); + return (ImageResolution.x * 0.5f) / std::tan(fovRad * 0.5f); + } + + void WriteCamerasTxt(const std::filesystem::path& outDir) + { + auto path = outDir / "cameras.txt"; + std::ofstream file(path); + if (!file.is_open()) + { + nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); + return; + } + + file << std::setprecision(12); + file << "# COLMAP camera intrinsics. Standard format (colmap.github.io/format.html).\n"; + file << "# OPENCV model: PARAMS = fx, fy, cx, cy, k1, k2, p1, p2 (pixels).\n"; + file << "# Camera list with one line of data per camera:\n"; + file << "# CAMERA_ID, MODEL, WIDTH, HEIGHT, PARAMS[]\n"; + file << "# Number of cameras: " << Frames.size() << "\n"; + + for (size_t i = 0; i < Frames.size(); ++i) + { + float fx = ComputeFocalLengthPixels(Frames[i]); + float fy = fx; + if (Frames[i].PixelAspectRatio > 0.0f) + fy = fx / Frames[i].PixelAspectRatio; + + // center_shift is in the same units as sensor_size (mm); convert to + // pixel offset on the principal point. See TrackToView.cpp:30 for the + // canonical centerShift / sensorSize relationship. + float cx = ImageResolution.x * 0.5f; + float cy = ImageResolution.y * 0.5f; + if (Frames[i].SensorSize.x > 0.0f) + cx += Frames[i].CenterShift.x * ImageResolution.x / Frames[i].SensorSize.x; + if (Frames[i].SensorSize.y > 0.0f) + cy += Frames[i].CenterShift.y * ImageResolution.y / Frames[i].SensorSize.y; + + float k1 = Frames[i].K1; + float k2 = Frames[i].K2; + + file << (i + 1) << " OPENCV " << ImageResolution.x << " " << ImageResolution.y << " " + << fx << " " << fy << " " << cx << " " << cy << " " + << k1 << " " << k2 << " 0 0\n"; + } + } + + void WriteImagesTxt(const std::filesystem::path& outDir) + { + auto path = outDir / "images.txt"; + std::ofstream file(path); + if (!file.is_open()) + { + nosEngine.LogE("RecordTrackCOLMAP: Cannot open %s", nos::PathToUtf8(path).c_str()); + return; + } + + file << std::setprecision(12); + file << "# COLMAP poses. Standard format (colmap.github.io/format.html).\n"; + file << "# Frame: RH, +X right, +Y down, +Z forward (camera looks along +Z).\n"; + file << "# (QW, QX, QY, QZ) is the world-to-camera rotation R_w2c.\n"; + file << "# (TX, TY, TZ) is the world-to-camera translation: t = -R_w2c * camera_world_position.\n"; + file << "# Recover camera position in the COLMAP world frame as: C = -R_w2c^T * t.\n"; + file << "# Image list with two lines of data per image:\n"; + file << "# IMAGE_ID, QW, QX, QY, QZ, TX, TY, TZ, CAMERA_ID, NAME\n"; + file << "# POINTS2D[] as (X, Y, POINT3D_ID)\n"; + file << "# Number of images: " << Frames.size() << "\n"; + + // M maps the SourceFrame to the COLMAP frame. Used to convert both the + // source-frame R_c2w and the source-frame camera position into COLMAP. + const glm::dmat3 M = convention::BasisChangeToColmap(SourceFrame); + const glm::dmat3 Minv = glm::inverse(M); + + for (size_t i = 0; i < Frames.size(); ++i) + { + auto& frame = Frames[i]; + + // Build R_c2w in the source frame, then conjugate by M to land in + // the COLMAP frame. Likewise frame the position. + glm::dmat3 R_c2w_src = convention::EulerToMat(SourceFrame, glm::dvec3(frame.Rotation)); + glm::dmat3 R_c2w_colmap = M * R_c2w_src * Minv; + glm::dvec3 pos_colmap = M * glm::dvec3(frame.Location); + + glm::dmat3 R_w2c = glm::transpose(R_c2w_colmap); + glm::dquat q_w2c = glm::quat_cast(R_w2c); + glm::dvec3 t = -R_w2c * pos_colmap; + + file << (i + 1) << " " + << q_w2c.w << " " << q_w2c.x << " " << q_w2c.y << " " << q_w2c.z << " " + << t.x << " " << t.y << " " << t.z << " " + << (i + 1) << " " + << "frame_" << std::setfill('0') << std::setw(6) << i << ".png\n"; + // Empty points line (required by COLMAP format) + file << "\n"; + } + } +}; + +void RegisterRecordTrackCOLMAP(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("RecordTrackCOLMAP"), RecordTrackCOLMAPContext, fn); +} + +} // namespace nos::track diff --git a/Plugins/nosTrack/Source/TrackMain.cpp b/Plugins/nosTrack/Source/TrackMain.cpp index c330165d..48b43fcc 100644 --- a/Plugins/nosTrack/Source/TrackMain.cpp +++ b/Plugins/nosTrack/Source/TrackMain.cpp @@ -15,12 +15,18 @@ enum TrackNode : int FreeD, UserTrack, AddTrack, + RecordTrackCOLMAP, + PlaybackTrackCOLMAP, + TrackTransform, Count }; void RegisterFreeDNode(nosNodeFunctions* functions); void RegisterController(nosNodeFunctions* functions); void RegisterAddTrack(nosNodeFunctions*); +void RegisterRecordTrackCOLMAP(nosNodeFunctions*); +void RegisterPlaybackTrackCOLMAP(nosNodeFunctions*); +void RegisterTrackTransform(nosNodeFunctions*); nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** outList) { @@ -40,7 +46,16 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** ou RegisterController(node); break; case TrackNode::AddTrack: - RegisterAddTrack(node); + RegisterAddTrack(node); + break; + case TrackNode::RecordTrackCOLMAP: + RegisterRecordTrackCOLMAP(node); + break; + case TrackNode::PlaybackTrackCOLMAP: + RegisterPlaybackTrackCOLMAP(node); + break; + case TrackNode::TrackTransform: + RegisterTrackTransform(node); break; } } diff --git a/Plugins/nosTrack/Source/TrackTransform.cpp b/Plugins/nosTrack/Source/TrackTransform.cpp new file mode 100644 index 00000000..0dabb1f1 --- /dev/null +++ b/Plugins/nosTrack/Source/TrackTransform.cpp @@ -0,0 +1,53 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include +#include + +#include "CoordinateFrameConv.h" + +namespace nos::track +{ + +void RegisterTrackTransform(nosNodeFunctions* funcs) +{ + funcs->ClassName = NOS_NAME("TrackTransform"); + funcs->ExecuteNode = [](void*, nosNodeExecuteParams* params) { + auto pins = GetPinValues(params); + auto ids = GetPinIds(params); + + auto* inTrack = flatbuffers::GetMutableRoot(pins[NOS_NAME("In")]); + auto source = *static_cast(pins[NOS_NAME("Source")]); + auto target = *static_cast(pins[NOS_NAME("Target")]); + float worldScale = *static_cast(pins[NOS_NAME("WorldScale")]); + + nos::sys::track::TTrack out; + inTrack->UnPackTo(&out); + + const glm::dmat3 S_src = convention::BasisMatrix(source); + const glm::dmat3 S_tgt = convention::BasisMatrix(target); + const glm::dmat3 M = S_tgt * glm::inverse(S_src); + + // Location: basis change, then uniform world-scale. Other Track fields + // (rotation, fov, focus, sensor_size, lens_distortion, ...) are unaffected. + const auto& inLoc = *inTrack->location(); + glm::dvec3 loc(inLoc.x(), inLoc.y(), inLoc.z()); + glm::dvec3 outLoc = M * loc * static_cast(worldScale); + out.location.mutate_x(static_cast(outLoc.x)); + out.location.mutate_y(static_cast(outLoc.y)); + out.location.mutate_z(static_cast(outLoc.z)); + + // Rotation: build in source frame, conjugate by M, extract in target frame. + const auto& inRot = *inTrack->rotation(); + glm::dmat3 R_src = convention::EulerToMat(source, glm::dvec3(inRot.x(), inRot.y(), inRot.z())); + glm::dmat3 R_tgt = M * R_src * glm::transpose(M); + glm::dvec3 outRotDeg = convention::MatToEuler(target, R_tgt); + out.rotation.mutate_x(static_cast(outRotDeg.x)); + out.rotation.mutate_y(static_cast(outRotDeg.y)); + out.rotation.mutate_z(static_cast(outRotDeg.z)); + + return nosEngine.SetPinValue(ids[NOS_NAME("Out")], nos::Buffer::From(out)); + }; +} + +} // namespace nos::track diff --git a/Plugins/nosTrack/Track.noscfg b/Plugins/nosTrack/Track.noscfg index 861423c0..3810af05 100644 --- a/Plugins/nosTrack/Track.noscfg +++ b/Plugins/nosTrack/Track.noscfg @@ -2,21 +2,27 @@ "info": { "id": { "name": "nos.track", - "version": "1.10.0" + "version": "1.11.0" }, "display_name": "Track", "category": "Virtual Studio", "dependencies": [ { "name": "nos.sys.track", - "version": "1.0" + "version": "1.1" } ] }, "node_definitions": [ "Config/FreeD.nosdef", "Config/UserTrack.nosdef", - "Config/AddTrack.nosdef" + "Config/AddTrack.nosdef", + "Config/RecordTrackCOLMAP.nosdef", + "Config/PlaybackTrackCOLMAP.nosdef", + "Config/TrackTransform.nosdef" + ], + "custom_types": [ + "Config/PlaybackMode.fbs" ], "defaults": [ "Config/Defaults.json" diff --git a/Plugins/nosUtilities/.nospub b/Plugins/nosUtilities/.nospub index 609d521e..9883f1c9 100644 --- a/Plugins/nosUtilities/.nospub +++ b/Plugins/nosUtilities/.nospub @@ -4,6 +4,7 @@ "*.noscfg", "Include/**", "Assets/**", + "Fonts/**", "Shaders/*.{hlsl,glsl,frag,vert,spv,comp}", "Binaries/*.{dll,dylib,so}" ], diff --git a/Plugins/nosUtilities/CMakeLists.txt b/Plugins/nosUtilities/CMakeLists.txt index 62db52c7..54c5910b 100644 --- a/Plugins/nosUtilities/CMakeLists.txt +++ b/Plugins/nosUtilities/CMakeLists.txt @@ -12,5 +12,17 @@ foreach(module_name_version ${MODULE_DEPENDENCIES}) list(APPEND MODULE_DEPENDENCIES_TARGETS ${DEP_${dep_idx}}) endforeach() -list(APPEND DEPENDENCIES stb nosUtilities_generated ${NOS_PLUGIN_SDK_TARGET} ${MODULE_DEPENDENCIES_TARGETS}) -nos_add_plugin("nosUtilities" "${DEPENDENCIES}" "") \ No newline at end of file +# FreeType is vendored as a submodule and built with no external dependencies +# so the text rendering node has a self-contained font rasterizer. +if (NOT TARGET freetype) + set(FT_DISABLE_ZLIB ON CACHE BOOL "" FORCE) + set(FT_DISABLE_BZIP2 ON CACHE BOOL "" FORCE) + set(FT_DISABLE_PNG ON CACHE BOOL "" FORCE) + set(FT_DISABLE_HARFBUZZ ON CACHE BOOL "" FORCE) + set(FT_DISABLE_BROTLI ON CACHE BOOL "" FORCE) + add_subdirectory(External/freetype EXCLUDE_FROM_ALL) + nos_group_targets("freetype" "External") +endif() + +list(APPEND DEPENDENCIES stb freetype nosUtilities_generated ${NOS_PLUGIN_SDK_TARGET} ${MODULE_DEPENDENCIES_TARGETS}) +nos_add_plugin("nosUtilities" "${DEPENDENCIES}" "") diff --git a/Plugins/nosUtilities/Config/ChannelViewer.fbs b/Plugins/nosUtilities/Config/ChannelViewer.fbs index b5de7eb1..7042da21 100644 --- a/Plugins/nosUtilities/Config/ChannelViewer.fbs +++ b/Plugins/nosUtilities/Config/ChannelViewer.fbs @@ -9,9 +9,3 @@ enum ChannelViewerChannels : uint { Cb = 5, Cr = 6 } - -enum ChannelViewerFormats : uint { - Rec_601 = 0, - Rec_709 = 1, - Rec_2020 = 2 -} diff --git a/Plugins/nosUtilities/Config/ChannelViewer.nosdef b/Plugins/nosUtilities/Config/ChannelViewer.nosdef index 4a002d10..2722cba5 100644 --- a/Plugins/nosUtilities/Config/ChannelViewer.nosdef +++ b/Plugins/nosUtilities/Config/ChannelViewer.nosdef @@ -31,10 +31,10 @@ }, { "name": "Format", - "type_name": "nos.utilities.ChannelViewerFormats", + "type_name": "nos.mediaio.ColorSpace", "show_as": "PROPERTY", "can_show_as": "PROPERTY_ONLY", - "data": "Rec_709", + "data": "REC709", "description": "Sets the input texture color space,\nRequired for correct YCbCr conversion" }, { diff --git a/Plugins/nosUtilities/Config/MultiBoundedQueue.nosdef b/Plugins/nosUtilities/Config/MultiBoundedQueue.nosdef new file mode 100644 index 00000000..df1ef4c8 --- /dev/null +++ b/Plugins/nosUtilities/Config/MultiBoundedQueue.nosdef @@ -0,0 +1,56 @@ +{ + "nodes": [ + { + "class_name": "MultiBoundedQueue", + "menu_info": { + "category": "Utilities", + "display_name": "Multi Bounded Queue", + "name_aliases": [ "data structure", "algorithm", "circular", "multi", "fifo" ] + }, + "node": { + "class_name": "MultiBoundedQueue", + "display_name": "Multi Bounded Queue", + "contents_type": "Job", + "description": "Bounded FIFO queue with one or more independent input/output channel pairs sharing a single bound. Right-click the node to add a channel, right-click an Input_X or Output_X pin to remove its channel.", + "pins": [ + { + "name": "Thread", + "type_name": "nos.exe", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Size", + "type_name": "uint", + "data": 2, + "max": 120, + "min": 1, + "show_as": "PROPERTY", + "can_show_as": "PROPERTY_ONLY" + }, + { + "name": "Alignment", + "description": "Used for creating memory-aligned buffers in memory", + "type_name": "uint", + "def": 0, + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Input_A", + "type_name": "nos.Generic", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Output_A", + "type_name": "nos.Generic", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "live": true + } + ] + } + } + ] +} diff --git a/Plugins/nosUtilities/Config/MultiLiveOut.nosdef b/Plugins/nosUtilities/Config/MultiLiveOut.nosdef new file mode 100644 index 00000000..36997973 --- /dev/null +++ b/Plugins/nosUtilities/Config/MultiLiveOut.nosdef @@ -0,0 +1,30 @@ +{ + "nodes": [ + { + "class_name": "MultiLiveOut", + "menu_info": { + "category": "Scheduling", + "display_name": "Multi Live Out" + }, + "node": { + "class_name": "MultiLiveOut", + "contents_type": "Job", + "pins": [ + { + "name": "Input_0", + "type_name": "nos.Generic", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Output_0", + "type_name": "nos.Generic", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "live": true + } + ] + } + } + ] +} diff --git a/Plugins/nosUtilities/Config/MultiRingBuffer.nosdef b/Plugins/nosUtilities/Config/MultiRingBuffer.nosdef new file mode 100644 index 00000000..3fa2c4c3 --- /dev/null +++ b/Plugins/nosUtilities/Config/MultiRingBuffer.nosdef @@ -0,0 +1,74 @@ +{ + "nodes": [ + { + "class_name": "MultiRingBuffer", + "menu_info": { + "category": "Utilities", + "display_name": "Multi Ring Buffer", + "name_aliases": [ "data structure", "algorithm", "circular", "multi" ] + }, + "node": { + "class_name": "MultiRingBuffer", + "display_name": "Multi Ring Buffer", + "contents_type": "Job", + "description": "Ring buffer with one or more independent input/output channel pairs sharing a single ring size. Right-click the node to add a channel, right-click an Input_X or Output_X pin to remove its channel.", + "pins": [ + { + "name": "Thread", + "type_name": "nos.exe", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Size", + "type_name": "uint", + "data": 2, + "max": 120, + "min": 1, + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Spare", + "type_name": "uint", + "data": 0, + "max": 119, + "min": 0, + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Alignment", + "description": "Used for creating memory-aligned buffers in memory", + "type_name": "uint", + "def": 0, + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + }, + { + "name": "Input_A", + "type_name": "nos.Generic", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Output_A", + "type_name": "nos.Generic", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "live": true + }, + { + "name": "RepeatWhenFilling", + "display_name": "Repeat When Filling", + "type_name": "bool", + "description": "Serves the last value while the buffer is being filled instead of waiting & resets the ring on restart", + "def": true, + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY" + } + ] + } + } + ] +} diff --git a/Plugins/nosUtilities/Config/ScheduleRequest.nosdef b/Plugins/nosUtilities/Config/ScheduleRequest.nosdef new file mode 100644 index 00000000..59042dbe --- /dev/null +++ b/Plugins/nosUtilities/Config/ScheduleRequest.nosdef @@ -0,0 +1,58 @@ +{ + "nodes": [ + { + "class_name": "ScheduleRequest", + "menu_info": { + "category": "Execution", + "display_name": "Schedule Request", + "aliases": [ "schedule", "on demand", "request" ] + }, + "node": { + "class_name": "ScheduleRequest", + "display_name": "Schedule Request", + "contents_type": "Job", + "description": "Drives an on-demand path. Each execution, and each path start, queues another\nschedule request so the path feeding Trigger keeps running. Wire the resource you\nwant scheduled into Sink.", + "pins": [ + { + "name": "Trigger", + "type_name": "nos.exe", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "Sink", + "type_name": "nos.Generic", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_ONLY" + }, + { + "name": "DeltaSeconds", + "display_name": "Delta Seconds", + "description": "Target time between path runs, as a rational x/y seconds. Default 1/60.", + "type_name": "nos.fb.vec2u", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": { "x": 1, "y": 60 } + }, + { + "name": "Importance", + "description": "Conflicting paths are controlled by the node of higher importance.", + "type_name": "uint", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1 + }, + { + "name": "TryAgainOnFailure", + "display_name": "Try Again On Failure", + "description": "If enabled, the request is retried when a node on the path returns an error.", + "type_name": "bool", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": true + } + ] + } + } + ] +} diff --git a/Plugins/nosUtilities/Config/TextRender.fbs b/Plugins/nosUtilities/Config/TextRender.fbs new file mode 100644 index 00000000..8bfe4e6f --- /dev/null +++ b/Plugins/nosUtilities/Config/TextRender.fbs @@ -0,0 +1,15 @@ +namespace nos.utilities; + +enum TextHAlign : uint +{ + LEFT = 0, + CENTER = 1, + RIGHT = 2, +} + +enum TextVAlign : uint +{ + TOP = 0, + MIDDLE = 1, + BOTTOM = 2, +} diff --git a/Plugins/nosUtilities/Config/TextRender.nosdef b/Plugins/nosUtilities/Config/TextRender.nosdef new file mode 100644 index 00000000..5ad61bde --- /dev/null +++ b/Plugins/nosUtilities/Config/TextRender.nosdef @@ -0,0 +1,275 @@ +{ + "nodes": [ + { + "class_name": "TextRender", + "menu_info": { + "category": "Utilities", + "display_name": "Text Render", + "name_aliases": [ "text", "font", "label", "caption", "subtitle", "string to texture" ] + }, + "node": { + "class_name": "TextRender", + "contents_type": "Job", + "description": "Renders text into a texture using a signed distance field font atlas.\nSupports multi-line text, word wrapping, outline, drop shadow and a text-box background.", + "pins": [ + { + "name": "Text", + "type_name": "string", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": "Text", + "description": "Text to render. Newlines and word wrapping are honored." + }, + { + "name": "FontSize", + "display_name": "Font Size", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Font" } + ], + "data": 64.0, + "min": 1.0, + "description": "Glyph height in pixels." + }, + { + "name": "Color", + "type_name": "nos.fb.vec4", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "visualizer": { + "type": "COLOR_PICKER" + }, + "data": { + "x": 1, + "y": 1, + "z": 1, + "w": 1 + }, + "description": "Fill color of the text." + }, + { + "name": "Opacity", + "type_name": "float", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "data": 1.0, + "min": 0.0, + "max": 1.0, + "description": "Global opacity multiplier applied to text, outline, shadow and background." + }, + { + "name": "StrokeColor", + "display_name": "Stroke Color", + "type_name": "nos.fb.vec4", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Outline" } + ], + "visualizer": { + "type": "COLOR_PICKER" + }, + "data": { + "x": 0, + "y": 0, + "z": 0, + "w": 1 + }, + "description": "Outline color. Only drawn when Stroke Width is greater than 0." + }, + { + "name": "StrokeWidth", + "display_name": "Stroke Width", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Outline" } + ], + "data": 0.0, + "min": 0.0, + "description": "Outline thickness in pixels. 0 disables the outline." + }, + { + "name": "ShadowColor", + "display_name": "Shadow Color", + "type_name": "nos.fb.vec4", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Shadow" } + ], + "visualizer": { + "type": "COLOR_PICKER" + }, + "data": { + "x": 0, + "y": 0, + "z": 0, + "w": 0 + }, + "description": "Drop shadow color. Alpha 0 disables the shadow." + }, + { + "name": "ShadowOffset", + "display_name": "Shadow Offset", + "type_name": "nos.fb.vec2", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Shadow" } + ], + "data": { + "x": 3, + "y": 3 + }, + "description": "Drop shadow offset in pixels (x right, y down)." + }, + { + "name": "ShadowSoftness", + "display_name": "Shadow Softness", + "type_name": "float", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Shadow" } + ], + "data": 3.0, + "min": 0.0, + "description": "Drop shadow edge blur in pixels." + }, + { + "name": "BackgroundColor", + "display_name": "Background Color", + "type_name": "nos.fb.vec4", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Background" } + ], + "visualizer": { + "type": "COLOR_PICKER" + }, + "data": { + "x": 0, + "y": 0, + "z": 0, + "w": 0.6 + }, + "description": "Color of the box drawn behind the text block.\nAlpha 0 disables the box; the frame stays transparent." + }, + { + "name": "BackgroundPadding", + "display_name": "Background Padding", + "type_name": "nos.fb.vec2", + "show_as": "PROPERTY", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Background" } + ], + "data": { + "x": 20, + "y": 10 + }, + "description": "Padding in pixels between the text and the background box edges." + }, + { + "name": "HorizontalAlign", + "display_name": "Horizontal Align", + "type_name": "nos.utilities.TextHAlign", + "show_as": "PROPERTY", + "can_show_as": "INPUT_OUTPUT_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Layout" } + ], + "data": "CENTER", + "description": "Horizontal anchor of the text block within the output." + }, + { + "name": "VerticalAlign", + "display_name": "Vertical Align", + "type_name": "nos.utilities.TextVAlign", + "show_as": "PROPERTY", + "can_show_as": "INPUT_OUTPUT_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Layout" } + ], + "data": "BOTTOM", + "description": "Vertical anchor of the text block within the output." + }, + { + "name": "Position", + "type_name": "nos.fb.vec2", + "show_as": "INPUT_PIN", + "can_show_as": "INPUT_PIN_OR_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Layout" } + ], + "data": { + "x": 0, + "y": -50 + }, + "description": "Extra offset in pixels applied on top of the alignment anchor." + }, + { + "name": "Resolution", + "type_name": "nos.fb.vec2u", + "show_as": "PROPERTY", + "can_show_as": "INPUT_OUTPUT_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Layout" } + ], + "data": { + "x": 1920, + "y": 1080 + }, + "description": "Output texture resolution.", + "visualizer": { + "type": "NAMED_VALUE", + "name": "nos.mediaio.ResolutionVisualizer" + } + }, + { + "name": "WrapWidth", + "display_name": "Wrap Width", + "type_name": "uint", + "show_as": "PROPERTY", + "can_show_as": "INPUT_OUTPUT_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Layout" } + ], + "data": 0, + "description": "Word-wrap width in characters.\n0 wraps to the output texture width." + }, + { + "name": "Font", + "type_name": "string", + "show_as": "PROPERTY", + "can_show_as": "INPUT_OUTPUT_PROPERTY", + "meta_data_map": [ + { "key": "Category", "value": "Font" } + ], + "data": "", + "description": "Path to a .ttf/.otf font file.\nLeave empty to use the bundled Roboto Mono font.", + "visualizer": { + "type": "FILE_PICKER", + "file_extensions": [ "ttf", "otf" ], + "file_picker_type": "OPEN" + } + }, + { + "name": "Output", + "type_name": "nos.sys.vulkan.Texture", + "show_as": "OUTPUT_PIN", + "can_show_as": "OUTPUT_PIN_ONLY", + "data": { + "unscaled": true + } + } + ] + } + } + ] +} diff --git a/Plugins/nosUtilities/External/freetype b/Plugins/nosUtilities/External/freetype new file mode 160000 index 00000000..7e0e56f8 --- /dev/null +++ b/Plugins/nosUtilities/External/freetype @@ -0,0 +1 @@ +Subproject commit 7e0e56f84fd53cf38378d33c8fc8f92d12ab9ac6 diff --git a/Plugins/nosUtilities/Fonts/LICENSE-RobotoMono.txt b/Plugins/nosUtilities/Fonts/LICENSE-RobotoMono.txt new file mode 100644 index 00000000..8e7df338 --- /dev/null +++ b/Plugins/nosUtilities/Fonts/LICENSE-RobotoMono.txt @@ -0,0 +1,93 @@ +Copyright 2015 The Roboto Mono Project Authors (https://github.com/googlefonts/robotomono) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +https://openfontlicense.org + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/Plugins/nosUtilities/Fonts/RobotoMono-Regular.ttf b/Plugins/nosUtilities/Fonts/RobotoMono-Regular.ttf new file mode 100644 index 00000000..f21d1d71 Binary files /dev/null and b/Plugins/nosUtilities/Fonts/RobotoMono-Regular.ttf differ diff --git a/Plugins/nosUtilities/Shaders/TextBox.frag b/Plugins/nosUtilities/Shaders/TextBox.frag new file mode 100644 index 00000000..c7d7e0a5 --- /dev/null +++ b/Plugins/nosUtilities/Shaders/TextBox.frag @@ -0,0 +1,19 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#version 450 +#extension GL_EXT_scalar_block_layout : enable + +layout(binding = 0, std430) uniform UBO +{ + vec2 Offset; + vec2 Size; + vec4 BoxColor; +} ubo; + +layout(location = 0) in vec2 uv; +layout(location = 0) out vec4 rt; + +void main() +{ + rt = ubo.BoxColor; +} diff --git a/Plugins/nosUtilities/Shaders/TextBox.vert b/Plugins/nosUtilities/Shaders/TextBox.vert new file mode 100644 index 00000000..2b2ac902 --- /dev/null +++ b/Plugins/nosUtilities/Shaders/TextBox.vert @@ -0,0 +1,29 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#version 450 +#extension GL_EXT_scalar_block_layout : enable + +layout(location = 0) out vec2 uv; + +layout(binding = 0, std430) uniform UBO +{ + vec2 Offset; // box top-left in 0..1 output coords (y down) + vec2 Size; // box size in 0..1 output coords + vec4 BoxColor; +} ubo; + +const vec2 pos[6] = + vec2[6]( + vec2(0.0, +1.0), + vec2(+1.0, +1.0), + vec2(0.0, 0.0), + vec2(0.0, 0.0), + vec2(+1.0, +1.0), + vec2(+1.0, 0.0)); + +void main() +{ + vec2 p = pos[gl_VertexIndex]; + gl_Position = vec4((p * ubo.Size * 2) + vec2(-1, -1) + ubo.Offset * 2, 0.0, 1.0); + uv = p; +} diff --git a/Plugins/nosUtilities/Shaders/TextGlyph.frag b/Plugins/nosUtilities/Shaders/TextGlyph.frag new file mode 100644 index 00000000..3b2246c0 --- /dev/null +++ b/Plugins/nosUtilities/Shaders/TextGlyph.frag @@ -0,0 +1,42 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#version 450 +#extension GL_EXT_scalar_block_layout : enable + +layout(binding = 1) uniform sampler2D Atlas; + +layout(binding = 0, std430) uniform UBO +{ + vec2 Offset; + vec2 Size; + vec4 AtlasRect; // xy = atlas uv min, zw = atlas uv extent + vec4 FillColor; + vec4 StrokeColor; + float StrokeWidth; // outline thickness in output pixels (0 = none) + float Softness; // extra edge blur in output pixels (drop shadow) + float PxRange; // output pixels spanned by one full signed-distance unit +} ubo; + +layout(location = 0) in vec2 uv; +layout(location = 0) out vec4 rt; + +void main() +{ + // uv (0,0) is the glyph top-left, matching the top-down atlas bitmaps. + vec2 atlasUv = ubo.AtlasRect.xy + uv * ubo.AtlasRect.zw; + float sd = texture(Atlas, atlasUv).r; // signed distance, 0.5 = glyph edge + + float distPx = (sd - 0.5) * ubo.PxRange; // signed distance, output pixels + float aa = 0.75 + ubo.Softness; + + float fillA = smoothstep(-aa, aa, distPx); + float outerA = smoothstep(-aa, aa, distPx + ubo.StrokeWidth); + + // Composite the fill over the stroke (stroke spans the whole silhouette). + float fa = ubo.FillColor.a * fillA; + float sa = ubo.StrokeColor.a * outerA; + float outA = fa + sa * (1.0 - fa); + vec3 outRGB = (ubo.FillColor.rgb * fa + ubo.StrokeColor.rgb * sa * (1.0 - fa)) / max(outA, 1e-5); + + rt = vec4(outRGB, outA); +} diff --git a/Plugins/nosUtilities/Shaders/TextGlyph.vert b/Plugins/nosUtilities/Shaders/TextGlyph.vert new file mode 100644 index 00000000..284e6cba --- /dev/null +++ b/Plugins/nosUtilities/Shaders/TextGlyph.vert @@ -0,0 +1,34 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#version 450 +#extension GL_EXT_scalar_block_layout : enable + +layout(location = 0) out vec2 uv; + +layout(binding = 0, std430) uniform UBO +{ + vec2 Offset; // glyph quad top-left in 0..1 output coords (y down) + vec2 Size; // glyph quad size in 0..1 output coords + vec4 AtlasRect; // xy = atlas uv min, zw = atlas uv extent + vec4 FillColor; + vec4 StrokeColor; + float StrokeWidth; + float Softness; + float PxRange; +} ubo; + +const vec2 pos[6] = + vec2[6]( + vec2(0.0, +1.0), + vec2(+1.0, +1.0), + vec2(0.0, 0.0), + vec2(0.0, 0.0), + vec2(+1.0, +1.0), + vec2(+1.0, 0.0)); + +void main() +{ + vec2 p = pos[gl_VertexIndex]; + gl_Position = vec4((p * ubo.Size * 2) + vec2(-1, -1) + ubo.Offset * 2, 0.0, 1.0); + uv = p; +} diff --git a/Plugins/nosUtilities/Source/ChannelViewer.cpp b/Plugins/nosUtilities/Source/ChannelViewer.cpp index a0d88f67..4fcc2e96 100644 --- a/Plugins/nosUtilities/Source/ChannelViewer.cpp +++ b/Plugins/nosUtilities/Source/ChannelViewer.cpp @@ -13,6 +13,41 @@ NOS_REGISTER_NAME_SPACED(Nos_Utilities_ChannelViewer, "nos.utilities.ChannelView namespace nos::utilities { +static nosResult MigrateNode(nosFbNodePtr nodePtr, nosBuffer* outBuffer) +{ + fb::TNode tNode; + nodePtr->UnPackTo(&tNode); + bool migrated = false; + for (auto& pin : tNode.pins) + { + if (!pin || pin->name != "Format") + continue; + bool legacyType = pin->type_name == "nos.utilities.ChannelViewerFormats" || + pin->type_name == "nos.fb.ChannelViewerFormats"; + const char* newValue = nullptr; + if (!pin->data.empty()) + { + std::string_view oldValue(reinterpret_cast(pin->data.data()), pin->data.size() - 1); + if (oldValue == "Rec_601") newValue = "REC601"; + else if (oldValue == "Rec_709") newValue = "REC709"; + else if (oldValue == "Rec_2020") newValue = "REC2020"; + } + if (!legacyType && !newValue) + continue; + pin->type_name = "nos.mediaio.ColorSpace"; + if (newValue) + { + std::string s = newValue; + pin->data = std::vector(s.c_str(), s.c_str() + s.size() + 1); + } + migrated = true; + } + if (!migrated) + return NOS_RESULT_SUCCESS; + *outBuffer = EngineBuffer::CopyFrom(tNode).Release(); + return NOS_RESULT_SUCCESS; +} + static nosResult ExecuteNode(void* ctx, nosNodeExecuteParams* pins) { auto values = GetPinValues(pins); @@ -25,7 +60,8 @@ static nosResult ExecuteNode(void* ctx, nosNodeExecuteParams* pins) glm::vec4 val{}; val[channel & 3] = 1; - constexpr glm::vec3 coeffs[3] = {{.299f, .587f, .114f}, {.2126f, .7152f, .0722f}, {.2627f, .678f, .0593f}}; + // Indexed by nos.mediaio.ColorSpace: REC709=0, REC601=1, REC2020=2 + constexpr glm::vec3 coeffs[3] = {{.2126f, .7152f, .0722f}, {.299f, .587f, .114f}, {.2627f, .678f, .0593f}}; glm::vec4 multipliers = glm::vec4(coeffs[format], channel > 3); std::vector bindings = { @@ -51,6 +87,7 @@ nosResult RegisterChannelViewer(nosNodeFunctions* out) { out->ClassName = NSN_Nos_Utilities_ChannelViewer; out->ExecuteNode = ExecuteNode; + out->MigrateNode = MigrateNode; fs::path root = nosEngine.Module->RootFolderPath; auto chViewerPath = (root / "Shaders" / "ChannelViewer.frag").generic_string(); diff --git a/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp b/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp new file mode 100644 index 00000000..9f433574 --- /dev/null +++ b/Plugins/nosUtilities/Source/MultiBoundedQueue.cpp @@ -0,0 +1,608 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#pragma once + +#include + +#include + +// External +#include +#include + +#include "MultiRing.h" +#include "Ring.h" +#include "nosUtil/Stopwatch.hpp" + +namespace nos::utilities +{ + +struct MultiBoundedQueueNodeContext : NodeContext +{ + static constexpr std::string_view CHANNEL_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + enum MenuCommandType : uint8_t + { + ADD_CHANNEL = 0, + REMOVE_CHANNEL = 1, + }; + + struct MenuCommand + { + MenuCommandType Type; + uint8_t Letter; + MenuCommand(uint32_t cmd) + { + Type = static_cast(cmd & 0xFF); + Letter = static_cast((cmd >> 8) & 0xFF); + } + MenuCommand(MenuCommandType type, uint8_t letter) : Type(type), Letter(letter) {} + operator uint32_t() const { return (Letter << 8) | Type; } + }; + + struct Channel + { + char Letter; + nos::Name InputName; + nos::Name OutputName; + uuid InputId{}; + uuid OutputId{}; + nos::TypeInfo TypeInfo; + MultiRing::Channel* RingChannel = nullptr; + std::atomic_bool IsOutLive = false; + bool NeedsRecreation = false; + + Channel(char letter) + : Letter(letter), + InputName((std::string("Input_") + letter).c_str()), + OutputName((std::string("Output_") + letter).c_str()), + TypeInfo(NSN_Generic) + { + } + }; + + std::map> Channels; + std::unordered_map PinIdToLetter; + MultiRing Ring; + // Channels popped since the last SendScheduleRequest. One producer run + // pushes one slot per live channel, so we must only schedule again once + // every live channel has been popped — otherwise schedule requests pile + // up by a factor of N (channels) per consumer tick. + std::set PoppedSinceLastSchedule; + + std::optional RequestedRingSize = std::nullopt; + + std::string GetName() const { return "MultiBoundedQueue"; } + + static std::optional ParseLetter(std::string_view pinName) + { + auto pos = pinName.find_last_of('_'); + if (pos == std::string::npos || pos + 2 != pinName.size()) + return std::nullopt; + char c = pinName[pos + 1]; + if (c < 'A' || c > 'Z') + return std::nullopt; + return c; + } + + static bool IsInputPin(std::string_view pinName) { return pinName.starts_with("Input_"); } + static bool IsOutputPin(std::string_view pinName) { return pinName.starts_with("Output_"); } + + MultiBoundedQueueNodeContext(nosFbNodePtr node) : NodeContext(node) + { + std::vector pinsToUnorphan; + for (auto* pin : *node->pins()) + { + auto pinNameSv = pin->name()->string_view(); + if (!IsInputPin(pinNameSv) && !IsOutputPin(pinNameSv)) + continue; + auto letter = ParseLetter(pinNameSv); + if (!letter) + continue; + + auto& channel = Channels[*letter]; + if (!channel) + channel = std::make_unique(*letter); + + if (IsInputPin(pinNameSv)) + channel->InputId = uuid(*pin->id()); + else + { + channel->OutputId = uuid(*pin->id()); + channel->IsOutLive = pin->live(); + } + PinIdToLetter[uuid(*pin->id())] = *letter; + + nos::Name typeName(pin->type_name()->c_str()); + if (typeName != NSN_Generic && channel->TypeInfo->TypeName == NSN_Generic) + channel->TypeInfo = nos::TypeInfo(typeName); + + if (auto orphanState = pin->orphan_state()) + if (orphanState->type() == fb::PinOrphanStateType::ORPHAN) + pinsToUnorphan.push_back(uuid(*pin->id())); + } + + for (auto& [_, ch] : Channels) + InitChannel(*ch); + + for (auto const& pinId : pinsToUnorphan) + SetPinOrphanState(pinId, fb::PinOrphanStateType::ACTIVE); + + AddPinValueWatcher(NSN_Size, [this](nos::Buffer const& newSize, std::optional oldVal) { + uint32_t size = *newSize.As(); + if (oldVal && oldVal == newSize) + return; + RequestRingResize(size); + }); + AddPinValueWatcher(NSN_Alignment, [this](nos::Buffer const& newAlignment, std::optional oldVal) { + bool any = false; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + if (ch->RingChannel->ResInterface->CheckNewResource(NSN_Alignment, newAlignment, oldVal)) + { + nosEngine.SendPathRestart(ch->InputId); + ch->NeedsRecreation = true; + any = true; + } + } + if (any) + { + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + } + }); + } + + ~MultiBoundedQueueNodeContext() override { Ring.Stop(); } + + void InitChannel(Channel& ch) + { + std::shared_ptr resource; + if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Buffer::GetFullyQualifiedName())) + resource = std::make_shared(); + else if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Texture::GetFullyQualifiedName())) + resource = std::make_shared(); + else + resource = std::make_shared(); + + ch.RingChannel = &Ring.AddChannel(ch.Letter, std::move(resource), &ch); + } + + Channel* GetChannelByPinId(uuid const& id) + { + auto it = PinIdToLetter.find(id); + if (it == PinIdToLetter.end()) + return nullptr; + auto chIt = Channels.find(it->second); + return chIt != Channels.end() ? chIt->second.get() : nullptr; + } + + void RequestRingResize(uint32_t size) + { + if (size == 0) + { + nosEngine.LogW((GetName() + " size cannot be 0").c_str()); + return; + } + if (Ring.Size == size && (!RequestedRingSize.has_value() || *RequestedRingSize == size)) + return; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + nosPathCommand ringSizeChange{.Event = NOS_RING_SIZE_CHANGE, .RingSize = size}; + nosEngine.SendPathCommand(ch->InputId, ringSizeChange); + } + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + SendPathRestart(); + RequestedRingSize = size; + } + + void SendPathRestart() + { + for (auto& [_, ch] : Channels) + nosEngine.SendPathRestart(ch->InputId); + } + + void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer value) override + { + auto sv = pinName.AsString(); + if (!IsInputPin(sv)) + return; + auto* ch = GetChannelByPinId(pinId); + if (!ch || !ch->RingChannel) + return; + if (ch->RingChannel->ResInterface->CheckNewResource(NSN_Input, value, std::nullopt)) + { + nosEngine.SendPathRestart(ch->InputId); + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + ch->NeedsRecreation = true; + } + } + + nosResult OnResolvePinDataTypes(nosResolvePinDataTypesParams* params) override + { + auto pinNameStr = nos::Name(params->InstigatorPinName).AsString(); + auto letter = ParseLetter(pinNameStr); + if (!letter) + return NOS_RESULT_FAILED; + auto chIt = Channels.find(*letter); + if (chIt == Channels.end()) + return NOS_RESULT_FAILED; + auto& ch = *chIt->second; + if (ch.TypeInfo->TypeName != NSN_Generic) + return NOS_RESULT_FAILED; + ch.TypeInfo = nos::TypeInfo(params->IncomingTypeName); + if (ch.RingChannel) + { + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + Ring.RemoveChannel(*letter); + ch.RingChannel = nullptr; + } + for (size_t i = 0; i < params->PinCount; i++) + { + auto& pinInfo = params->Pins[i]; + if (pinInfo.Id == ch.InputId || pinInfo.Id == ch.OutputId) + pinInfo.OutResolvedTypeName = ch.TypeInfo->TypeName; + } + return NOS_RESULT_SUCCESS; + } + + void OnPinUpdated(const nosPinUpdate*) override + { + for (auto& [_, ch] : Channels) + if (!ch->RingChannel) + InitChannel(*ch); + } + + void OnNodeUpdated(nosNodeUpdate const* update) override + { + if (update->Type == NOS_NODE_UPDATE_PIN_DELETED) + { + auto it = PinIdToLetter.find(update->PinDeleted); + if (it == PinIdToLetter.end()) + return; + char letter = it->second; + PinIdToLetter.erase(it); + auto chIt = Channels.find(letter); + if (chIt == Channels.end()) + return; + auto& ch = *chIt->second; + bool inputAlive = PinIdToLetter.contains(ch.InputId); + bool outputAlive = PinIdToLetter.contains(ch.OutputId); + if (!inputAlive && !outputAlive) + { + if (ch.RingChannel) + { + Ring.RemoveChannel(letter); + ch.RingChannel = nullptr; + } + Channels.erase(chIt); + } + } + else if (update->Type == NOS_NODE_UPDATE_PIN_CREATED) + { + auto* pin = update->PinCreated; + auto sv = pin->name()->string_view(); + if (!IsInputPin(sv) && !IsOutputPin(sv)) + return; + auto letter = ParseLetter(sv); + if (!letter) + return; + auto& chPtr = Channels[*letter]; + if (!chPtr) + chPtr = std::make_unique(*letter); + if (IsInputPin(sv)) + chPtr->InputId = uuid(*pin->id()); + else + { + chPtr->OutputId = uuid(*pin->id()); + chPtr->IsOutLive = pin->live(); + } + PinIdToLetter[uuid(*pin->id())] = *letter; + if (!chPtr->RingChannel) + InitChannel(*chPtr); + } + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + if (Channels.empty() || Ring.Exit) + return NOS_RESULT_FAILED; + + NodeExecuteParams pins(params); + uint32_t requestedSize = *pins.GetPinData(NSN_Size); + + struct Gathered + { + Channel* NodeCh; + MultiRing::Channel* RingCh; + void* Input; + }; + std::vector gathered; + gathered.reserve(Channels.size()); + std::vector wantedRings; + wantedRings.reserve(Channels.size()); + + uint32_t maxRequired = requestedSize; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel || ch->RingChannel->Resources.empty() || !ch->TypeInfo) + continue; + auto it = pins.find(ch->InputName); + if (it == pins.end()) + continue; + void* input = ch->RingChannel->ResInterface->GetPinInfo(it->second, false); + if (!input) + continue; + auto [required, _] = ch->RingChannel->ResInterface->GetRequiredRingSize(input, requestedSize); + if (required > maxRequired) + maxRequired = required; + gathered.push_back({ch.get(), ch->RingChannel, input}); + wantedRings.push_back(ch->RingChannel); + } + if (gathered.empty()) + { + SendScheduleRequest(0); + return NOS_RESULT_FAILED; + } + + if (Ring.Size != maxRequired) + { + RequestRingResize(maxRequired); + return NOS_RESULT_FAILED; + } + + std::vector slots; + if (!Ring.BeginPushSubset(100, wantedRings, slots)) + return Ring.Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; + + for (size_t i = 0; i < gathered.size(); ++i) + { + auto& g = gathered[i]; + auto* slot = slots[i].second; + g.RingCh->ResInterface->Push(slot, g.Input, params, + NOS_NAME_STATIC("MultiBoundedQueue"), false); + if (!g.NodeCh->IsOutLive) + { + ChangePinLiveness(g.NodeCh->OutputName, true); + g.NodeCh->IsOutLive = true; + } + } + + Ring.EndPushAll(slots); + return NOS_RESULT_SUCCESS; + } + + nosResult CopyFrom(nosCopyInfo* cpy) override + { + auto* ch = GetChannelByPinId(cpy->ID); + if (!ch || !ch->RingChannel || Ring.Exit) + return NOS_RESULT_FAILED; + if (!ch->IsOutLive) + return NOS_RESULT_SUCCESS; + + ResourceInterface::ResourceBase* slot; + { + ScopedProfilerEvent _({.Name = "Wait For Filled Slot"}); + slot = Ring.BeginPop(*ch->RingChannel, 100); + } + if (!slot) + return Ring.Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; + + // Propagate the slot resource's descriptor onto the output pin before + // Copy reads cpy->PinData as the destination — otherwise the GPU copy + // targets the stale (default-sized) output descriptor. + nos::Buffer outPinVal; + if (ch->RingChannel->ResInterface->BeginCopyFrom(slot, *cpy->PinData, outPinVal)) + nosEngine.SetPinValueByName(NodeId, ch->OutputName, outPinVal); + + ch->RingChannel->ResInterface->Copy(slot, cpy, NodeId); + + cpy->CopyFromOptions.ShouldSetSourceFrameNumber = true; + cpy->FrameNumber = slot->FrameNumber; + + Ring.EndPop(*ch->RingChannel, slot); + + PoppedSinceLastSchedule.insert(ch->Letter); + size_t liveCount = 0; + for (auto& [_, c] : Channels) + if (c->IsOutLive) + ++liveCount; + if (PoppedSinceLastSchedule.size() >= liveCount) + { + SendScheduleRequest(1); + PoppedSinceLastSchedule.clear(); + } + return NOS_RESULT_SUCCESS; + } + + void OnEndFrame(uuid const& pinId, nosEndFrameCause cause) override + { + if (cause != NOS_END_FRAME_FAILED) + return; + auto* ch = GetChannelByPinId(pinId); + if (!ch) + return; + if (pinId == ch->OutputId) + return; + if (!ch->IsOutLive) + return; + ChangePinLiveness(ch->OutputName, false); + ch->IsOutLive = false; + } + + void SendScheduleRequest(uint32_t count, bool reset = false) const + { + nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = count, .Reset = reset}; + nosEngine.ScheduleNode(&schedule); + } + + void OnPathCommand(const nosPathCommand* command) override + { + switch (command->Event) + { + case NOS_RING_SIZE_CHANGE: + if (command->RingSize == 0) + return; + RequestedRingSize = command->RingSize; + nosEngine.SetPinValue(*GetPinId(NSN_Size), nos::Buffer::From(command->RingSize)); + break; + default: return; + } + } + + void OnPathStop() override + { + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + } + + void OnPathStart() override + { + if (Channels.empty()) + return; + + PoppedSinceLastSchedule.clear(); + + Ring.ResetAll(false); + + if (RequestedRingSize) + { + Ring.ResizeAll(*RequestedRingSize); + for (auto& [_, ch] : Channels) + ch->NeedsRecreation = false; + RequestedRingSize = std::nullopt; + } + for (auto& [_, ch] : Channels) + { + if (ch->NeedsRecreation && ch->RingChannel) + { + Ring.RecreateChannelResources(*ch->RingChannel); + ch->NeedsRecreation = false; + } + } + + size_t totalSchedule = 0; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + if (ch->RingChannel->Resources.empty()) + { + totalSchedule = std::max(totalSchedule, 1); + continue; + } + auto emptySlotCount = Ring.WritePoolSize(*ch->RingChannel); + totalSchedule = std::max(totalSchedule, emptySlotCount); + ch->RingChannel->ResInterface->OnPathStart(); + } + Ring.Start(); + if (totalSchedule > 0) + { + nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = (uint32_t)totalSchedule}; + nosEngine.ScheduleNode(&schedule); + } + } + + void OnNodeMenuRequested(nosContextMenuRequestPtr request) override + { + flatbuffers::FlatBufferBuilder fbb; + std::vector items = { + nos::CreateContextMenuItemDirect(fbb, "Add Channel", MenuCommand(ADD_CHANNEL, 0))}; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnPinMenuRequested(nos::Name pinName, nosContextMenuRequestPtr request) override + { + auto sv = pinName.AsString(); + if (!IsInputPin(sv) && !IsOutputPin(sv)) + return; + auto letter = ParseLetter(sv); + if (!letter) + return; + if (Channels.size() <= 1) + return; + flatbuffers::FlatBufferBuilder fbb; + std::vector items = {nos::CreateContextMenuItemDirect( + fbb, "Remove Channel", MenuCommand(REMOVE_CHANNEL, static_cast(*letter)))}; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnMenuCommand(uuid const& itemID, uint32_t cmd) override + { + auto command = MenuCommand(cmd); + switch (command.Type) + { + case ADD_CHANNEL: + { + char newLetter = 0; + for (char c : CHANNEL_LETTERS) + { + if (!Channels.contains(c)) + { + newLetter = c; + break; + } + } + if (newLetter == 0) + { + SetNodeStatusMessage("Maximum number of channels reached", fb::NodeStatusMessageType::WARNING); + return; + } + + fb::TPin inPin; + inPin.id = uuid(nosEngine.GenerateID()); + inPin.name = std::string("Input_") + newLetter; + inPin.type_name = "nos.Generic"; + inPin.show_as = fb::ShowAs::INPUT_PIN; + inPin.can_show_as = fb::CanShowAs::INPUT_PIN_ONLY; + + fb::TPin outPin; + outPin.id = uuid(nosEngine.GenerateID()); + outPin.name = std::string("Output_") + newLetter; + outPin.type_name = "nos.Generic"; + outPin.show_as = fb::ShowAs::OUTPUT_PIN; + outPin.can_show_as = fb::CanShowAs::OUTPUT_PIN_ONLY; + outPin.live = true; + + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_add.emplace_back(std::make_unique(std::move(inPin))); + update.pins_to_add.emplace_back(std::make_unique(std::move(outPin))); + flatbuffers::FlatBufferBuilder fbb; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + break; + } + case REMOVE_CHANNEL: + { + char letter = static_cast(command.Letter); + auto it = Channels.find(letter); + if (it == Channels.end()) + return; + auto& ch = *it->second; + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_delete = {ch.InputId, ch.OutputId}; + flatbuffers::FlatBufferBuilder fbb; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + break; + } + } + } +}; + +nosResult RegisterMultiBoundedQueue(nosNodeFunctions* functions) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("MultiBoundedQueue"), MultiBoundedQueueNodeContext, functions) + return NOS_RESULT_SUCCESS; +} + +} // namespace nos::utilities diff --git a/Plugins/nosUtilities/Source/MultiLiveOut.cpp b/Plugins/nosUtilities/Source/MultiLiveOut.cpp new file mode 100644 index 00000000..c4d08d88 --- /dev/null +++ b/Plugins/nosUtilities/Source/MultiLiveOut.cpp @@ -0,0 +1,189 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include + +namespace nos::utilities +{ + +struct MultiLiveOutNode : NodeContext +{ + MultiLiveOutNode(nosFbNodePtr node) : NodeContext(node) + { + for (auto* pin : *node->pins()) + { + SetPinOrphanState(*pin->id(), nos::fb::PinOrphanStateType::ACTIVE); + auto index = GetPinIndex(pin->name()->string_view()); + if (!index) + { + nosEngine.LogE("Failed to parse index from pin name: %s", pin->name()->c_str()); + continue; + } + if (pin->show_as() == nosFbShowAs::OUTPUT_PIN) + IndexToPairs[*index].second = uuid(*pin->id()); + else + IndexToPairs[*index].first = uuid(*pin->id()); + } + } + + void OnNodeUpdated(nosNodeUpdate const* update) override + { + if (update->Type == NOS_NODE_UPDATE_PIN_CREATED) + { + auto* pin = update->PinCreated; + auto index = GetPinIndex(pin->name()->string_view()); + if (!index) + return; + if (pin->show_as() == nosFbShowAs::OUTPUT_PIN) + IndexToPairs[*index].second = uuid(*pin->id()); + else + IndexToPairs[*index].first = uuid(*pin->id()); + } + else if (update->Type == NOS_NODE_UPDATE_PIN_DELETED) + { + for (auto it = IndexToPairs.begin(); it != IndexToPairs.end(); ++it) + { + if (it->second.first == update->PinDeleted || it->second.second == update->PinDeleted) + { + IndexToPairs.erase(it); + break; + } + } + } + } + + void OnMenuRequested(nosContextMenuRequestPtr request) override + { + flatbuffers::FlatBufferBuilder fbb; + std::vector> items; + if (*request->item_id() == NodeId) + items.push_back(nos::CreateContextMenuItemDirect(fbb, "Add New Pair", 1)); + else + { + auto* pin = GetPin(*request->item_id()); + if (!pin) + return; + if (pin->Name == NOS_NAME("Input_0") || pin->Name == NOS_NAME("Output_0")) + return; + items.push_back(nos::CreateContextMenuItemDirect(fbb, "Remove Pair", 1)); + } + HandleEvent(CreateAppEvent( + fbb, CreateAppContextMenuUpdate( + fbb, request->item_id(), request->pos(), request->instigator(), fbb.CreateVector(items)))); + } + + void OnMenuCommand(uuid const& itemID, uint32_t cmd) override + { + flatbuffers::FlatBufferBuilder fbb; + if (itemID == NodeId) + { + int index = 0; + for (; index < (int)IndexToPairs.size(); index++) + { + if (!IndexToPairs.contains(index)) + break; + } + fb::TPin outPin; + outPin.id = uuid(nosEngine.GenerateID()); + outPin.name = "Output_" + std::to_string(index); + outPin.type_name = NOS_NAME("nos.Generic"); + outPin.live = true; + outPin.show_as = fb::ShowAs::OUTPUT_PIN; + outPin.can_show_as = fb::CanShowAs::OUTPUT_PIN_ONLY; + + fb::TPin inPin; + inPin.id = uuid(nosEngine.GenerateID()); + inPin.name = "Input_" + std::to_string(index); + inPin.type_name = NOS_NAME("nos.Generic"); + inPin.show_as = fb::ShowAs::INPUT_PIN; + inPin.can_show_as = fb::CanShowAs::INPUT_PIN_ONLY; + + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_add.emplace_back(std::make_unique(std::move(outPin))); + update.pins_to_add.emplace_back(std::make_unique(std::move(inPin))); + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + IndexToPairs[index] = {uuid(inPin.id), uuid(outPin.id)}; + } + else + { + auto* pin = GetPin(itemID); + if (!pin) + return; + auto index = GetPinIndex(pin->Name.AsString()); + if (!index) + { + nosEngine.LogE("Failed to parse index from pin name: %s", pin->Name.AsCStr()); + return; + } + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_delete = {IndexToPairs[*index].first, IndexToPairs[*index].second}; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + IndexToPairs.erase(*index); + } + } + + nosResult OnResolvePinDataTypes(nosResolvePinDataTypesParams* params) override + { + auto pinName = nos::Name(params->InstigatorPinName).AsString(); + auto index = GetPinIndex(pinName); + if (!index.has_value()) + { + strcpy(params->OutErrorMessage, "Failed to parse pin index from pin name."); + return NOS_RESULT_FAILED; + } + auto const& [firstId, secondId] = IndexToPairs[*index]; + for (size_t i = 0; i < params->PinCount; i++) + { + auto& pin = params->Pins[i]; + if (pin.Id == firstId || pin.Id == secondId) + pin.OutResolvedTypeName = params->IncomingTypeName; + else + pin.OutResolvedTypeName = NOS_NAME("nos.Generic"); + } + return NOS_RESULT_SUCCESS; + } + + std::optional GetPinIndex(std::string_view pinName) const + { + auto indexPos = pinName.find_last_of('_'); + if (indexPos == std::string::npos) + return std::nullopt; + try + { + return std::stoi(std::string(pinName.substr(indexPos + 1))); + } + catch (...) + { + nosEngine.LogE("Failed to parse index from pin name: %s", std::string(pinName).c_str()); + return std::nullopt; + } + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + for (auto const& [_, idPair] : IndexToPairs) + { + for (size_t i = 0; i < params->PinCount; ++i) + { + auto& pin = params->Pins[i]; + if (pin.Id == idPair.first && pin.Data) + { + nosEngine.SetPinValue(idPair.second, *pin.Data); + break; + } + } + } + return NOS_RESULT_SUCCESS; + } + + std::unordered_map> IndexToPairs; +}; + +nosResult RegisterMultiLiveOut(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("MultiLiveOut"), MultiLiveOutNode, fn) + return NOS_RESULT_SUCCESS; +} + +} // namespace nos::utilities diff --git a/Plugins/nosUtilities/Source/MultiRing.h b/Plugins/nosUtilities/Source/MultiRing.h new file mode 100644 index 00000000..c2d0ba14 --- /dev/null +++ b/Plugins/nosUtilities/Source/MultiRing.h @@ -0,0 +1,256 @@ +/* + * Copyright MediaZ Teknoloji A.S. All Rights Reserved. + */ + +#pragma once + +#include "Ring.h" + +namespace nos +{ + +// Ring that holds N independent channels under a single mutex / CV pair. +// Each channel still owns its own slot pools and Resources, but every push, +// pop, resize and reset goes through the shared synchronization, so an +// N-channel batch push is a single lock acquisition, not N. +struct MultiRing +{ + struct Channel + { + std::shared_ptr ResInterface; + std::vector> Resources; + std::deque WritePool; + std::deque ReadPool; + void* UserData = nullptr; + }; + + std::map> Channels; + std::mutex Mutex; + std::condition_variable WriteCV; + std::condition_variable ReadCV; + std::atomic_bool Exit = true; + uint32_t Size = 0; + + ~MultiRing() { Stop(); } + + void Stop() + { + { + std::unique_lock lock(Mutex); + Exit = true; + } + WriteCV.notify_all(); + ReadCV.notify_all(); + } + + void Start() + { + std::unique_lock lock(Mutex); + Exit = false; + } + + void AllocateChannelResourcesUnlocked(Channel& ch) + { + ch.WritePool.clear(); + ch.ReadPool.clear(); + ch.Resources.clear(); + for (uint32_t i = 0; i < Size; ++i) + { + auto res = ch.ResInterface->CreateResource(); + if (!res) + { + nosEngine.LogE("Failed to create resource for multi ring buffer."); + ch.Resources.clear(); + ch.WritePool.clear(); + ch.ReadPool.clear(); + Exit = true; + return; + } + ch.Resources.push_back(res); + ch.WritePool.push_back(res.get()); + } + } + + Channel& AddChannel(char key, std::shared_ptr resInterface, void* userData = nullptr) + { + std::unique_lock lock(Mutex); + auto& ch = Channels[key]; + if (!ch) + ch = std::make_unique(); + ch->ResInterface = std::move(resInterface); + ch->UserData = userData; + if (Size == 0) + Size = 1; + AllocateChannelResourcesUnlocked(*ch); + return *ch; + } + + void RemoveChannel(char key) + { + std::unique_lock lock(Mutex); + Channels.erase(key); + } + + void RecreateChannelResources(Channel& ch) + { + std::unique_lock lock(Mutex); + AllocateChannelResourcesUnlocked(ch); + } + + void ResizeAll(uint32_t newSize) + { + std::unique_lock lock(Mutex); + Size = newSize; + for (auto& [_, ch] : Channels) + AllocateChannelResourcesUnlocked(*ch); + } + + bool AreAllChannelsValid() + { + std::unique_lock lock(Mutex); + if (Channels.empty()) + return false; + for (auto& [_, ch] : Channels) + if (ch->Resources.empty()) + return false; + return true; + } + + // Move slots between pools for every channel. fill=false: read→write. + void ResetAll(bool fill) + { + std::unique_lock lock(Mutex); + for (auto& [_, ch] : Channels) + { + auto& from = fill ? ch->WritePool : ch->ReadPool; + auto& to = fill ? ch->ReadPool : ch->WritePool; + while (!from.empty()) + { + auto* slot = from.front(); + from.pop_front(); + ch->ResInterface->Reset(slot); + to.push_back(slot); + } + } + } + + // If this channel is full and its read pool is non-empty, hand one slot + // back to the write pool so the producer can start pushing again. + void MoveOneReadToWriteIfFull(Channel& ch) + { + std::unique_lock lock(Mutex); + if (ch.ReadPool.size() != ch.Resources.size() || ch.ReadPool.empty()) + return; + auto* slot = ch.ReadPool.front(); + ch.ReadPool.pop_front(); + ch.WritePool.push_back(slot); + } + + bool IsFull(Channel const& ch) + { + std::unique_lock lock(Mutex); + return ch.ReadPool.size() == ch.Resources.size(); + } + + bool IsEmpty(Channel const& ch) + { + std::unique_lock lock(Mutex); + return ch.ReadPool.empty(); + } + + size_t WritePoolSize(Channel const& ch) + { + std::unique_lock lock(Mutex); + return ch.WritePool.size(); + } + + size_t ReadPoolSize(Channel const& ch) + { + std::unique_lock lock(Mutex); + return ch.ReadPool.size(); + } + + using SlotPair = std::pair; + + // Atomically pop one slot from each requested channel's WritePool. + // Waits until every requested channel has at least one slot, or + // timeout/exit. The caller-supplied list typically excludes channels + // that don't have valid input data this frame. + bool BeginPushSubset(uint64_t timeoutMs, + std::vector const& wanted, + std::vector& outSlots) + { + std::unique_lock lock(Mutex); + auto pred = [&] { + if (Exit) + return true; + if (wanted.empty()) + return false; + for (auto* ch : wanted) + if (ch->WritePool.empty()) + return false; + return true; + }; + if (!WriteCV.wait_for(lock, std::chrono::milliseconds(timeoutMs), pred)) + return false; + if (Exit) + return false; + outSlots.clear(); + outSlots.reserve(wanted.size()); + for (auto* ch : wanted) + { + auto* slot = ch->WritePool.front(); + ch->WritePool.pop_front(); + outSlots.emplace_back(ch, slot); + } + return true; + } + + void EndPushAll(std::vector const& slots) + { + { + std::unique_lock lock(Mutex); + for (auto& [ch, slot] : slots) + ch->ReadPool.push_back(slot); + } + ReadCV.notify_all(); + } + + void CancelPushAll(std::vector const& slots) + { + { + std::unique_lock lock(Mutex); + for (auto& [ch, slot] : slots) + { + slot->FrameNumber = 0; + ch->WritePool.push_front(slot); + } + } + WriteCV.notify_all(); + } + + ResourceInterface::ResourceBase* BeginPop(Channel& ch, uint64_t timeoutMs) + { + std::unique_lock lock(Mutex); + if (!ReadCV.wait_for(lock, std::chrono::milliseconds(timeoutMs), + [&] { return !ch.ReadPool.empty() || Exit; })) + return nullptr; + if (Exit) + return nullptr; + auto* slot = ch.ReadPool.front(); + ch.ReadPool.pop_front(); + return slot; + } + + void EndPop(Channel& ch, ResourceInterface::ResourceBase* slot) + { + { + std::unique_lock lock(Mutex); + slot->FrameNumber = 0; + ch.WritePool.push_back(slot); + } + WriteCV.notify_all(); + } +}; + +} // namespace nos diff --git a/Plugins/nosUtilities/Source/MultiRingBuffer.cpp b/Plugins/nosUtilities/Source/MultiRingBuffer.cpp new file mode 100644 index 00000000..2c446299 --- /dev/null +++ b/Plugins/nosUtilities/Source/MultiRingBuffer.cpp @@ -0,0 +1,731 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#pragma once + +#include + +#include + +// External +#include +#include + +#include "MultiRing.h" +#include "Ring.h" +#include "nosUtil/Stopwatch.hpp" + +namespace nos::utilities +{ + +struct MultiRingBufferNodeContext : NodeContext +{ + using RingMode = RingNodeBase::RingMode; + using OnRestartType = RingNodeBase::OnRestartType; + + static constexpr std::string_view CHANNEL_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + enum MenuCommandType : uint8_t + { + ADD_CHANNEL = 0, + REMOVE_CHANNEL = 1, + }; + + struct MenuCommand + { + MenuCommandType Type; + uint8_t Letter; + MenuCommand(uint32_t cmd) + { + Type = static_cast(cmd & 0xFF); + Letter = static_cast((cmd >> 8) & 0xFF); + } + MenuCommand(MenuCommandType type, uint8_t letter) : Type(type), Letter(letter) {} + operator uint32_t() const { return (Letter << 8) | Type; } + }; + + struct Channel + { + char Letter; + nos::Name InputName; + nos::Name OutputName; + uuid InputId{}; + uuid OutputId{}; + nos::TypeInfo TypeInfo; + MultiRing::Channel* RingChannel = nullptr; + std::atomic_bool IsOutLive = false; + ResourceInterface::ResourceBase* LastPopped = nullptr; + bool NeedsRecreation = false; + std::size_t RemainingRepeatableCount = 0; + + Channel(char letter) + : Letter(letter), + InputName((std::string("Input_") + letter).c_str()), + OutputName((std::string("Output_") + letter).c_str()), + TypeInfo(NSN_Generic) + { + } + }; + + std::map> Channels; + std::unordered_map PinIdToLetter; + MultiRing Ring; + // Channels popped since the last SendScheduleRequest. One producer run + // pushes one slot per live channel, so we must only schedule again once + // every live channel has been popped — otherwise schedule requests pile + // up by a factor of N (channels) per consumer tick. + std::set PoppedSinceLastSchedule; + + OnRestartType OnRestart = OnRestartType::WAIT_UNTIL_FULL; + std::optional RequestedRingSize = std::nullopt; + std::atomic Mode = RingMode::CONSUME; + std::condition_variable ModeCV; + std::mutex ModeMutex; + std::atomic_bool RepeatWhenFilling = false; + + std::string GetName() const { return "MultiRingBuffer"; } + + static std::optional ParseLetter(std::string_view pinName) + { + auto pos = pinName.find_last_of('_'); + if (pos == std::string::npos || pos + 2 != pinName.size()) + return std::nullopt; + char c = pinName[pos + 1]; + if (c < 'A' || c > 'Z') + return std::nullopt; + return c; + } + + static bool IsInputPin(std::string_view pinName) { return pinName.starts_with("Input_"); } + static bool IsOutputPin(std::string_view pinName) { return pinName.starts_with("Output_"); } + + MultiRingBufferNodeContext(nosFbNodePtr node) : NodeContext(node) + { + std::vector pinsToUnorphan; + for (auto* pin : *node->pins()) + { + auto pinNameSv = pin->name()->string_view(); + if (!IsInputPin(pinNameSv) && !IsOutputPin(pinNameSv)) + continue; + auto letter = ParseLetter(pinNameSv); + if (!letter) + continue; + + auto& channel = Channels[*letter]; + if (!channel) + channel = std::make_unique(*letter); + + if (IsInputPin(pinNameSv)) + channel->InputId = uuid(*pin->id()); + else + { + channel->OutputId = uuid(*pin->id()); + channel->IsOutLive = pin->live(); + } + PinIdToLetter[uuid(*pin->id())] = *letter; + + nos::Name typeName(pin->type_name()->c_str()); + if (typeName != NSN_Generic && channel->TypeInfo->TypeName == NSN_Generic) + channel->TypeInfo = nos::TypeInfo(typeName); + + if (auto orphanState = pin->orphan_state()) + if (orphanState->type() == fb::PinOrphanStateType::ORPHAN) + pinsToUnorphan.push_back(uuid(*pin->id())); + } + + for (auto& [_, ch] : Channels) + InitChannel(*ch); + + for (auto const& pinId : pinsToUnorphan) + SetPinOrphanState(pinId, fb::PinOrphanStateType::ACTIVE); + + AddPinValueWatcher(NSN_Size, [this](nos::Buffer const& newSize, std::optional oldVal) { + uint32_t size = *newSize.As(); + if (oldVal && oldVal == newSize) + return; + RequestRingResize(size); + }); + AddPinValueWatcher(NSN_Alignment, [this](nos::Buffer const& newAlignment, std::optional oldVal) { + bool any = false; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + if (ch->RingChannel->ResInterface->CheckNewResource(NSN_Alignment, newAlignment, oldVal)) + { + nosEngine.SendPathRestart(ch->InputId); + ch->NeedsRecreation = true; + any = true; + } + } + if (any) + { + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + } + }); + AddPinValueWatcher(NOS_NAME_STATIC("RepeatWhenFilling"), + [this](nos::Buffer const& newVal, std::optional oldVal) { + RepeatWhenFilling = *newVal.As(); + }); + } + + ~MultiRingBufferNodeContext() override + { + for (auto& [_, ch] : Channels) + NOS_SOFT_CHECK(ch->LastPopped == nullptr); + Ring.Stop(); + } + + void InitChannel(Channel& ch) + { + std::shared_ptr resource; + if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Buffer::GetFullyQualifiedName())) + resource = std::make_shared(); + else if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Texture::GetFullyQualifiedName())) + resource = std::make_shared(); + else + resource = std::make_shared(); + + ch.RingChannel = &Ring.AddChannel(ch.Letter, std::move(resource), &ch); + } + + Channel* GetChannelByPinId(uuid const& id) + { + auto it = PinIdToLetter.find(id); + if (it == PinIdToLetter.end()) + return nullptr; + auto chIt = Channels.find(it->second); + return chIt != Channels.end() ? chIt->second.get() : nullptr; + } + + void SeedOutputPin(Channel& ch) + { + if (!ch.RingChannel || ch.RingChannel->Resources.empty()) + return; + auto* base = ch.RingChannel->Resources[0].get(); + if (!base) + return; + if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Buffer::GetFullyQualifiedName())) + { + if (auto* res = ResourceInterface::GetResource(base)) + nosEngine.SetPinValueByName(NodeId, ch.OutputName, res->VkRes.ToPinData()); + } + else if (ch.TypeInfo->TypeName == NOS_NAME(sys::vulkan::Texture::GetFullyQualifiedName())) + { + if (auto* res = ResourceInterface::GetResource(base)) + { + sys::vulkan::TTexture texDef = vkss::ConvertTextureInfo(res->VkRes); + texDef.unscaled = true; + nosEngine.SetPinValueByName(NodeId, ch.OutputName, nos::Buffer::From(texDef)); + } + } + } + + void RequestRingResize(uint32_t size) + { + if (size == 0) + { + nosEngine.LogW((GetName() + " size cannot be 0").c_str()); + return; + } + if (Ring.Size == size && (!RequestedRingSize.has_value() || *RequestedRingSize == size)) + return; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + nosPathCommand ringSizeChange{.Event = NOS_RING_SIZE_CHANGE, .RingSize = size}; + nosEngine.SendPathCommand(ch->InputId, ringSizeChange); + } + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + SendPathRestart(); + RequestedRingSize = size; + } + + void SendPathRestart() + { + for (auto& [_, ch] : Channels) + nosEngine.SendPathRestart(ch->InputId); + } + + void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer value) override + { + auto sv = pinName.AsString(); + if (!IsInputPin(sv)) + return; + auto* ch = GetChannelByPinId(pinId); + if (!ch || !ch->RingChannel) + return; + if (ch->RingChannel->ResInterface->CheckNewResource(NSN_Input, value, std::nullopt)) + { + nosEngine.SendPathRestart(ch->InputId); + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + ch->NeedsRecreation = true; + } + } + + nosResult OnResolvePinDataTypes(nosResolvePinDataTypesParams* params) override + { + auto pinNameStr = nos::Name(params->InstigatorPinName).AsString(); + auto letter = ParseLetter(pinNameStr); + if (!letter) + return NOS_RESULT_FAILED; + auto chIt = Channels.find(*letter); + if (chIt == Channels.end()) + return NOS_RESULT_FAILED; + auto& ch = *chIt->second; + if (ch.TypeInfo->TypeName != NSN_Generic) + return NOS_RESULT_FAILED; + ch.TypeInfo = nos::TypeInfo(params->IncomingTypeName); + // Drop the Generic-fallback ring channel so OnPinUpdated re-inits with the resolved type. + if (ch.RingChannel) + { + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + Ring.RemoveChannel(*letter); + ch.RingChannel = nullptr; + } + for (size_t i = 0; i < params->PinCount; i++) + { + auto& pinInfo = params->Pins[i]; + if (pinInfo.Id == ch.InputId || pinInfo.Id == ch.OutputId) + pinInfo.OutResolvedTypeName = ch.TypeInfo->TypeName; + } + return NOS_RESULT_SUCCESS; + } + + void OnPinUpdated(const nosPinUpdate*) override + { + for (auto& [_, ch] : Channels) + if (!ch->RingChannel) + InitChannel(*ch); + } + + void OnNodeUpdated(nosNodeUpdate const* update) override + { + if (update->Type == NOS_NODE_UPDATE_PIN_DELETED) + { + auto it = PinIdToLetter.find(update->PinDeleted); + if (it == PinIdToLetter.end()) + return; + char letter = it->second; + PinIdToLetter.erase(it); + auto chIt = Channels.find(letter); + if (chIt == Channels.end()) + return; + auto& ch = *chIt->second; + bool inputAlive = PinIdToLetter.contains(ch.InputId); + bool outputAlive = PinIdToLetter.contains(ch.OutputId); + if (!inputAlive && !outputAlive) + { + if (ch.RingChannel) + { + Ring.RemoveChannel(letter); + ch.RingChannel = nullptr; + } + Channels.erase(chIt); + } + } + else if (update->Type == NOS_NODE_UPDATE_PIN_CREATED) + { + auto* pin = update->PinCreated; + auto sv = pin->name()->string_view(); + if (!IsInputPin(sv) && !IsOutputPin(sv)) + return; + auto letter = ParseLetter(sv); + if (!letter) + return; + auto& chPtr = Channels[*letter]; + if (!chPtr) + chPtr = std::make_unique(*letter); + if (IsInputPin(sv)) + chPtr->InputId = uuid(*pin->id()); + else + { + chPtr->OutputId = uuid(*pin->id()); + chPtr->IsOutLive = pin->live(); + } + PinIdToLetter[uuid(*pin->id())] = *letter; + if (!chPtr->RingChannel) + InitChannel(*chPtr); + } + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + if (Channels.empty() || Ring.Exit) + return NOS_RESULT_FAILED; + + NodeExecuteParams pins(params); + uint32_t requestedSize = *pins.GetPinData(NSN_Size); + + struct Gathered + { + Channel* NodeCh; + MultiRing::Channel* RingCh; + void* Input; + }; + std::vector gathered; + gathered.reserve(Channels.size()); + std::vector wantedRings; + wantedRings.reserve(Channels.size()); + + uint32_t maxRequired = requestedSize; + std::string adjustMessage; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel || ch->RingChannel->Resources.empty() || !ch->TypeInfo) + continue; + auto it = pins.find(ch->InputName); + if (it == pins.end()) + continue; + void* input = ch->RingChannel->ResInterface->GetPinInfo(it->second, true); + if (!input) + continue; + auto [required, message] = ch->RingChannel->ResInterface->GetRequiredRingSize(input, requestedSize); + if (required > maxRequired) + { + maxRequired = required; + adjustMessage = message; + } + gathered.push_back({ch.get(), ch->RingChannel, input}); + wantedRings.push_back(ch->RingChannel); + } + if (gathered.empty()) + { + SendScheduleRequest(0); + return NOS_RESULT_FAILED; + } + + bool effectiveSizeAdjusted = maxRequired != requestedSize; + ClearNodeStatusMessages(); + if (effectiveSizeAdjusted) + SetNodeStatusMessage(adjustMessage, fb::NodeStatusMessageType::WARNING); + + if (Ring.Size != maxRequired) + { + RequestRingResize(maxRequired); + if (effectiveSizeAdjusted) + nosEngine.LogW("%s", adjustMessage.c_str()); + return NOS_RESULT_FAILED; + } + + std::vector slots; + if (!Ring.BeginPushSubset(100, wantedRings, slots)) + return Ring.Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; + + // Push outside the lock — Vulkan command recording can be slow. + for (size_t i = 0; i < gathered.size(); ++i) + { + auto& g = gathered[i]; + auto* slot = slots[i].second; + g.RingCh->ResInterface->Push(slot, g.Input, params, + NOS_NAME_STATIC("MultiRingBuffer"), true); + if (!g.NodeCh->IsOutLive) + { + ChangePinLiveness(g.NodeCh->OutputName, true); + g.NodeCh->IsOutLive = true; + } + } + + Ring.EndPushAll(slots); + + if (Mode == RingMode::FILL) + { + bool isFillComplete = true; + for (auto* rc : wantedRings) + if (Ring.WritePoolSize(*rc) != 0) + { + isFillComplete = false; + break; + } + if (isFillComplete) + { + Mode = RingMode::CONSUME; + ModeCV.notify_all(); + } + } + + return NOS_RESULT_SUCCESS; + } + + nosResult CopyFrom(nosCopyInfo* cpy) override + { + auto* ch = GetChannelByPinId(cpy->ID); + if (!ch || !ch->RingChannel || Ring.Exit) + return NOS_RESULT_FAILED; + if (!ch->IsOutLive) + return NOS_RESULT_SUCCESS; + + // EndPop the previous frame's slot before popping a new one. We can't + // rely on OnEndFrame: the engine only fires it on the path's primary + // source pin, so live secondary outputs (e.g. a second channel feeding + // the same consumer) never receive it. By the time the consumer asks + // for the next frame on this pin, it's done with the previous one. + if (ch->LastPopped) + { + Ring.EndPop(*ch->RingChannel, ch->LastPopped); + ch->LastPopped = nullptr; + } + + if (OnRestart == OnRestartType::WAIT_UNTIL_FULL && RepeatWhenFilling) + { + if (ch->RemainingRepeatableCount > 0) + { + ch->RingChannel->ResInterface->OnRepeatPinValue(cpy); + ch->RemainingRepeatableCount--; + return NOS_RESULT_SUCCESS; + } + } + else if (Mode == RingMode::FILL) + { + std::unique_lock lock(ModeMutex); + if (!ModeCV.wait_for(lock, std::chrono::milliseconds(100), + [this] { return Mode != RingMode::FILL; })) + return NOS_RESULT_PENDING; + } + + ResourceInterface::ResourceBase* slot; + { + ScopedProfilerEvent _({.Name = "Wait For Filled Slot"}); + slot = Ring.BeginPop(*ch->RingChannel, 100); + } + if (!slot) + return Ring.Exit ? NOS_RESULT_FAILED : NOS_RESULT_PENDING; + + nos::Buffer outPinVal; + bool changePinValue = ch->RingChannel->ResInterface->BeginCopyFrom(slot, *cpy->PinData, outPinVal); + if (changePinValue) + nosEngine.SetPinValueByName(NodeId, ch->OutputName, outPinVal); + + ch->RingChannel->ResInterface->WaitForDownloadToEnd(slot, "MultiRingBuffer", NodeName.AsString(), cpy); + + cpy->CopyFromOptions.ShouldSetSourceFrameNumber = true; + cpy->FrameNumber = slot->FrameNumber; + + ch->LastPopped = slot; + + PoppedSinceLastSchedule.insert(ch->Letter); + size_t liveCount = 0; + for (auto& [_, c] : Channels) + if (c->IsOutLive) + ++liveCount; + if (PoppedSinceLastSchedule.size() >= liveCount) + { + SendScheduleRequest(1); + PoppedSinceLastSchedule.clear(); + } + return NOS_RESULT_SUCCESS; + } + + void OnEndFrame(uuid const& pinId, nosEndFrameCause cause) override + { + auto* ch = GetChannelByPinId(pinId); + if (!ch) + return; + + if (cause == NOS_END_FRAME_FAILED) + { + if (pinId == ch->OutputId) + return; + if (!ch->IsOutLive) + return; + ChangePinLiveness(ch->OutputName, false); + ch->IsOutLive = false; + } + // EndPop happens at the start of the next CopyFrom for this channel + // rather than here, because OnEndFrame is unreliable for secondary + // live outputs. + } + + void SendScheduleRequest(uint32_t count, bool reset = false) const + { + nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = count, .Reset = reset}; + nosEngine.ScheduleNode(&schedule); + } + + void OnPathCommand(const nosPathCommand* command) override + { + switch (command->Event) + { + case NOS_RING_SIZE_CHANGE: + if (command->RingSize == 0) + return; + RequestedRingSize = command->RingSize; + nosEngine.SetPinValue(*GetPinId(NSN_Size), nos::Buffer::From(command->RingSize)); + break; + default: return; + } + } + + void OnPathStop() override + { + if (OnRestart == OnRestartType::WAIT_UNTIL_FULL) + Mode = RingMode::FILL; + for (auto& [_, ch] : Channels) + { + if (ch->LastPopped && ch->RingChannel) + { + Ring.EndPop(*ch->RingChannel, ch->LastPopped); + ch->LastPopped = nullptr; + } + } + Ring.Stop(); + PoppedSinceLastSchedule.clear(); + } + + void OnPathStart() override + { + if (Channels.empty()) + return; + + PoppedSinceLastSchedule.clear(); + + if (OnRestart == OnRestartType::RESET || RepeatWhenFilling) + Ring.ResetAll(false); + else + { + for (auto& [_, ch] : Channels) + if (ch->RingChannel) + Ring.MoveOneReadToWriteIfFull(*ch->RingChannel); + } + + if (RequestedRingSize) + { + Ring.ResizeAll(*RequestedRingSize); + for (auto& [_, ch] : Channels) + ch->NeedsRecreation = false; + RequestedRingSize = std::nullopt; + } + for (auto& [_, ch] : Channels) + { + if (ch->NeedsRecreation && ch->RingChannel) + { + Ring.RecreateChannelResources(*ch->RingChannel); + ch->NeedsRecreation = false; + } + } + + size_t totalSchedule = 0; + for (auto& [_, ch] : Channels) + { + if (!ch->RingChannel) + continue; + if (ch->RingChannel->Resources.empty()) + { + totalSchedule = std::max(totalSchedule, 1); + continue; + } + auto emptySlotCount = Ring.WritePoolSize(*ch->RingChannel); + if (RepeatWhenFilling) + ch->RemainingRepeatableCount = std::max(emptySlotCount, (size_t)1) - 1; + totalSchedule = std::max(totalSchedule, emptySlotCount); + ch->RingChannel->ResInterface->OnPathStart(); + SeedOutputPin(*ch); + } + Ring.Start(); + if (totalSchedule > 0) + { + nosScheduleNodeParams schedule{.NodeId = NodeId, .AddScheduleCount = (uint32_t)totalSchedule}; + nosEngine.ScheduleNode(&schedule); + } + } + + void OnNodeMenuRequested(nosContextMenuRequestPtr request) override + { + flatbuffers::FlatBufferBuilder fbb; + std::vector items = { + nos::CreateContextMenuItemDirect(fbb, "Add Channel", MenuCommand(ADD_CHANNEL, 0))}; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnPinMenuRequested(nos::Name pinName, nosContextMenuRequestPtr request) override + { + auto sv = pinName.AsString(); + if (!IsInputPin(sv) && !IsOutputPin(sv)) + return; + auto letter = ParseLetter(sv); + if (!letter) + return; + if (Channels.size() <= 1) + return; + flatbuffers::FlatBufferBuilder fbb; + std::vector items = {nos::CreateContextMenuItemDirect( + fbb, "Remove Channel", MenuCommand(REMOVE_CHANNEL, static_cast(*letter)))}; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnMenuCommand(uuid const& itemID, uint32_t cmd) override + { + auto command = MenuCommand(cmd); + switch (command.Type) + { + case ADD_CHANNEL: + { + char newLetter = 0; + for (char c : CHANNEL_LETTERS) + { + if (!Channels.contains(c)) + { + newLetter = c; + break; + } + } + if (newLetter == 0) + { + SetNodeStatusMessage("Maximum number of channels reached", fb::NodeStatusMessageType::WARNING); + return; + } + + fb::TPin inPin; + inPin.id = uuid(nosEngine.GenerateID()); + inPin.name = std::string("Input_") + newLetter; + inPin.type_name = "nos.Generic"; + inPin.show_as = fb::ShowAs::INPUT_PIN; + inPin.can_show_as = fb::CanShowAs::INPUT_PIN_ONLY; + + fb::TPin outPin; + outPin.id = uuid(nosEngine.GenerateID()); + outPin.name = std::string("Output_") + newLetter; + outPin.type_name = "nos.Generic"; + outPin.show_as = fb::ShowAs::OUTPUT_PIN; + outPin.can_show_as = fb::CanShowAs::OUTPUT_PIN_ONLY; + outPin.live = true; + + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_add.emplace_back(std::make_unique(std::move(inPin))); + update.pins_to_add.emplace_back(std::make_unique(std::move(outPin))); + flatbuffers::FlatBufferBuilder fbb; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + break; + } + case REMOVE_CHANNEL: + { + char letter = static_cast(command.Letter); + auto it = Channels.find(letter); + if (it == Channels.end()) + return; + auto& ch = *it->second; + nos::TPartialNodeUpdate update; + update.node_id = NodeId; + update.pins_to_delete = {ch.InputId, ch.OutputId}; + flatbuffers::FlatBufferBuilder fbb; + HandleEvent(CreateAppEvent(fbb, nos::CreatePartialNodeUpdate(fbb, &update))); + break; + } + } + } +}; + +nosResult RegisterMultiRingBuffer(nosNodeFunctions* functions) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("MultiRingBuffer"), MultiRingBufferNodeContext, functions) + return NOS_RESULT_SUCCESS; +} + +} // namespace nos::utilities diff --git a/Plugins/nosUtilities/Source/ScheduleRequest.cpp b/Plugins/nosUtilities/Source/ScheduleRequest.cpp new file mode 100644 index 00000000..4b21f93b --- /dev/null +++ b/Plugins/nosUtilities/Source/ScheduleRequest.cpp @@ -0,0 +1,80 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include + +namespace nos::utilities +{ + +// Drives an on-demand path: each execution (and each path start) queues another +// schedule request, so the path feeding the Trigger pin keeps running. Wire the +// thing you want scheduled into Sink. Ported from nos.flow (dev branch). +struct ScheduleRequestNode : NodeContext +{ + bool TryAgainOnFailure = true; + nosVec2u DeltaSeconds = { 1, 60 }; + uint32_t Importance = 1; + + ScheduleRequestNode(nosFbNodePtr node) : NodeContext(node) + { + if (node->pins()) + for (auto* pin : *node->pins()) + { + auto* data = pin->data(); + if (data && data->size()) + ReadPin(nos::Name(pin->name()->c_str()), data->data()); + } + } + + void ReadPin(nos::Name name, const void* data) + { + if (name == NOS_NAME("DeltaSeconds")) + DeltaSeconds = *static_cast(data); + else if (name == NOS_NAME("Importance")) + Importance = *static_cast(data); + else if (name == NOS_NAME("TryAgainOnFailure")) + TryAgainOnFailure = *static_cast(data); + } + + void OnPinValueChanged(nos::Name pinName, uuid const& pinId, nosBuffer value) override + { + ReadPin(pinName, value.Data); + } + + void GetScheduleInfo(nosScheduleInfo* info) override + { + info->Type = NOS_SCHEDULE_TYPE_ON_DEMAND; + info->DeltaSeconds = DeltaSeconds; + info->Importance = Importance; + } + + void ScheduleOnce() + { + nosScheduleNodeParams params{ .NodeId = NodeId, .AddScheduleCount = 1, .Reset = false }; + nosEngine.ScheduleNode(¶ms); + } + + nosResult ExecuteNode(nosNodeExecuteParams* params) override + { + ScheduleOnce(); + return NOS_RESULT_SUCCESS; + } + + void OnPathStart() override + { + ScheduleOnce(); + } + + void OnEndFrame(uuid const& pinId, nosEndFrameCause cause) override + { + if (TryAgainOnFailure && cause == NOS_END_FRAME_FAILED) + ScheduleOnce(); + } +}; + +nosResult RegisterScheduleRequest(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NOS_NAME("ScheduleRequest"), ScheduleRequestNode, fn); + return NOS_RESULT_SUCCESS; +} + +} // namespace nos::utilities diff --git a/Plugins/nosUtilities/Source/Sink.cpp b/Plugins/nosUtilities/Source/Sink.cpp index 043982f3..28294d7e 100644 --- a/Plugins/nosUtilities/Source/Sink.cpp +++ b/Plugins/nosUtilities/Source/Sink.cpp @@ -5,6 +5,7 @@ // stl #include +#include #include #include "Sink_generated.h" @@ -18,6 +19,33 @@ constexpr uint64_t VULKAN_TIMEOUT_BEFORE_LEAK = struct SinkNode : NodeContext { + enum MenuCommandType : uint8_t + { + ADD_INPUT = 0, + REMOVE_INPUT = 1, + }; + + struct MenuCommand + { + MenuCommandType Type; + uint8_t InputIndex; + MenuCommand(uint32_t cmd) { + Type = static_cast(cmd & 0xFF); + InputIndex = static_cast((cmd >> 8) & 0xFF); + } + MenuCommand(MenuCommandType type, uint8_t inputIndex) : Type(type), InputIndex(inputIndex) {} + operator uint32_t() const { return (InputIndex << 8) | Type; } + }; + + static const std::unordered_set& StaticPinNames() + { + static const std::unordered_set names = { + "InExe", "Sink Input", "Sink FPS", "HasGPUWork", "GPUFrameBuffering", + "AcceptsRepeat", "SinkMode", "LatencyBudget" + }; + return names; + } + std::mutex Mutex; std::atomic ShouldStop = false; std::atomic Fps = 1000.0f / 60.0f; @@ -31,9 +59,27 @@ struct SinkNode : NodeContext std::optional> GPUFrameSyncEvents = std::nullopt; size_t GPUFrameBuffering = 1; uint64_t CurrentGPUEventIndex = 0; + std::vector DynamicInputs; SinkNode(nosFbNodePtr inNode) : NodeContext(inNode) { + std::list pinsToUnorphan; + for (auto i = 0; i < inNode->pins()->size(); i++) + { + auto pin = inNode->pins()->Get(i); + if (pin->show_as() != fb::ShowAs::INPUT_PIN) + continue; + if (StaticPinNames().contains(pin->name()->string_view())) + continue; + DynamicInputs.push_back(*pin->id()); + if (auto orphanState = pin->orphan_state()) + { + if (orphanState->type() == fb::PinOrphanStateType::ORPHAN) + pinsToUnorphan.push_back(*pin->id()); + } + } + for (auto const& pinId : pinsToUnorphan) + SetPinOrphanState(pinId, fb::PinOrphanStateType::ACTIVE); AddPinValueWatcher(NOS_NAME("HasGPUWork"), [this](nosBuffer const& newVal, std::optional oldValue) { bool hasGpuWork = *static_cast(newVal.Data); @@ -255,6 +301,93 @@ struct SinkNode : NodeContext } } + void OnNodeMenuRequested(nosContextMenuRequestPtr request) override + { + uint32_t cmd = MenuCommand(ADD_INPUT, 0); + flatbuffers::FlatBufferBuilder fbb; + std::vector items = { + nos::CreateContextMenuItemDirect(fbb, "Add Sink", cmd, nullptr) + }; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnPinMenuRequested(nos::Name pinName, nosContextMenuRequestPtr request) override + { + if (StaticPinNames().contains(pinName.AsString())) + return; + auto pinId = GetPinId(pinName); + if (!pinId) + return; + auto it = std::find(DynamicInputs.begin(), DynamicInputs.end(), *pinId); + if (it == DynamicInputs.end()) + return; + auto index = std::distance(DynamicInputs.begin(), it); + uint32_t cmd = MenuCommand(REMOVE_INPUT, static_cast(index)); + flatbuffers::FlatBufferBuilder fbb; + std::vector items = { + nos::CreateContextMenuItemDirect(fbb, "Remove Input", cmd, nullptr) + }; + HandleEvent(CreateAppEvent(fbb, app::CreateAppContextMenuUpdateDirect( + fbb, request->item_id(), request->pos(), request->instigator(), &items))); + } + + void OnMenuCommand(uuid const& itemID, uint32_t cmd) override + { + auto command = MenuCommand(cmd); + switch (command.Type) + { + case ADD_INPUT: + { + std::string pinName; + for (size_t i = 2;; i++) + { + auto candidate = "Sink Input " + std::to_string(i); + if (!GetPinId(nos::Name(candidate))) + { + pinName = std::move(candidate); + break; + } + } + flatbuffers::FlatBufferBuilder fbb; + uuid pinId = nosEngine.GenerateID(); + std::vector pins = { + fb::CreatePinDirect(fbb, &pinId, pinName.c_str(), "nos.Generic", + fb::ShowAs::INPUT_PIN, fb::CanShowAs::INPUT_PIN_ONLY) + }; + HandleEvent(CreateAppEvent(fbb, CreatePartialNodeUpdateDirect(fbb, &NodeId, ClearFlags::NONE, 0, &pins))); + break; + } + case REMOVE_INPUT: + { + if (command.InputIndex >= DynamicInputs.size()) + return; + auto pinId = DynamicInputs[command.InputIndex]; + flatbuffers::FlatBufferBuilder fbb; + std::vector pinsToRemove = { *&pinId }; + HandleEvent(CreateAppEvent(fbb, CreatePartialNodeUpdateDirect(fbb, &NodeId, ClearFlags::NONE, &pinsToRemove))); + break; + } + } + } + + void OnNodeUpdated(nosNodeUpdate const* update) override + { + if (update->Type == NOS_NODE_UPDATE_PIN_DELETED) + { + std::erase_if(DynamicInputs, [&](auto id) { return id == update->PinDeleted; }); + } + else if (update->Type == NOS_NODE_UPDATE_PIN_CREATED) + { + auto* pin = update->PinCreated; + if (pin->show_as() != fb::ShowAs::INPUT_PIN) + return; + if (StaticPinNames().contains(pin->name()->string_view())) + return; + DynamicInputs.push_back(*pin->id()); + } + } + void GetScheduleInfo(nosScheduleInfo* info) override { info->Type = NOS_SCHEDULE_TYPE_ON_DEMAND; diff --git a/Plugins/nosUtilities/Source/TextRender.cpp b/Plugins/nosUtilities/Source/TextRender.cpp new file mode 100644 index 00000000..37548680 --- /dev/null +++ b/Plugins/nosUtilities/Source/TextRender.cpp @@ -0,0 +1,699 @@ +// Copyright MediaZ Teknoloji A.S. All Rights Reserved. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include FT_FREETYPE_H +#include FT_MODULE_H + +namespace nos::utilities +{ +namespace fs = std::filesystem; + +NOS_REGISTER_NAME(TextRender) +NOS_REGISTER_NAME(TextGlyph_Pass) +NOS_REGISTER_NAME(TextGlyph_Frag) +NOS_REGISTER_NAME(TextGlyph_Vert) +NOS_REGISTER_NAME(TextBox_Pass) +NOS_REGISTER_NAME(TextBox_Frag) +NOS_REGISTER_NAME(TextBox_Vert) + +NOS_REGISTER_NAME(Text) +NOS_REGISTER_NAME(FontSize) +NOS_REGISTER_NAME(Color) +NOS_REGISTER_NAME(Opacity) +NOS_REGISTER_NAME(StrokeColor) +NOS_REGISTER_NAME(StrokeWidth) +NOS_REGISTER_NAME(ShadowColor) +NOS_REGISTER_NAME(ShadowOffset) +NOS_REGISTER_NAME(ShadowSoftness) +NOS_REGISTER_NAME(BackgroundColor) +NOS_REGISTER_NAME(BackgroundPadding) +NOS_REGISTER_NAME(HorizontalAlign) +NOS_REGISTER_NAME(VerticalAlign) +NOS_REGISTER_NAME(Position) +NOS_REGISTER_NAME(Resolution) +NOS_REGISTER_NAME(WrapWidth) +NOS_REGISTER_NAME(Font) +NOS_REGISTER_NAME(Output) + +NOS_REGISTER_NAME(Offset) +NOS_REGISTER_NAME(Size) +NOS_REGISTER_NAME(AtlasRect) +NOS_REGISTER_NAME(FillColor) +NOS_REGISTER_NAME(Softness) +NOS_REGISTER_NAME(PxRange) +NOS_REGISTER_NAME(Atlas) +NOS_REGISTER_NAME(BoxColor) + +// ASCII printable range packed into the SDF atlas. +constexpr uint32_t FIRST_CHAR = 32; +constexpr uint32_t LAST_CHAR = 126; +constexpr uint32_t GLYPH_COUNT = LAST_CHAR - FIRST_CHAR + 1; +// Reference size the atlas is rasterized at; the SDF is scaled by the shader. +constexpr float REF_PIXEL_SIZE = 72.0f; +// SDF spread in reference pixels: how far signed-distance data extends from the +// glyph edge. Caps the usable outline thickness and shadow softness. +constexpr int SDF_SPREAD = 16; +constexpr int ATLAS_WIDTH = 1024; +constexpr int ATLAS_PADDING = 2; +// Upper bound on draw calls per execution to keep pathological inputs cheap. +constexpr size_t MAX_DRAWN_GLYPHS = 8192; + +struct Glyph +{ + int AtlasX = 0, AtlasY = 0; // top-left in atlas pixels + int Width = 0, Height = 0; // bitmap size in reference pixels + float BearingLeft = 0; // pen-origin to bitmap left, reference pixels + float BearingTop = 0; // baseline to bitmap top, reference pixels + float Advance = 0; // horizontal advance, reference pixels + bool HasBitmap = false; +}; + +struct TextRenderNode : NodeContext +{ + FT_Library Library = nullptr; + nosResourceShareInfo AtlasTex{}; + + std::array Glyphs{}; + int AtlasW = 0, AtlasH = 0; + float RefLineHeight = REF_PIXEL_SIZE * 1.2f; + float RefAscender = REF_PIXEL_SIZE; + + // Path the current atlas was built from; empty means "bundled font". + std::optional BuiltFontPath; + bool AtlasValid = false; + + TextRenderNode(nosFbNodePtr node) : NodeContext(node) + { + if (FT_Init_FreeType(&Library)) + { + Library = nullptr; + nosEngine.LogE("TextRender: failed to initialize FreeType"); + return; + } + FT_Int spread = SDF_SPREAD; + FT_Property_Set(Library, "sdf", "spread", &spread); + } + + ~TextRenderNode() override + { + DestroyAtlas(); + if (Library) + FT_Done_FreeType(Library); + } + + void DestroyAtlas() + { + if (AtlasTex.Memory.Handle) + nosVulkan->DestroyResource(&AtlasTex); + AtlasTex = {}; + AtlasValid = false; + } + + std::string ResolveFontPath(const char* fontPin) const + { + if (fontPin && fontPin[0] != '\0') + return fontPin; + fs::path root = nosEngine.Module->RootFolderPath; + return (root / "Fonts" / "RobotoMono-Regular.ttf").generic_string(); + } + + // Rasterizes the printable ASCII range into a single-channel SDF atlas + // and uploads it as a texture. Returns false on failure. + bool BuildAtlas(const std::string& fontPath) + { + if (!Library) + return false; + + FT_Face face = nullptr; + if (FT_New_Face(Library, fontPath.c_str(), 0, &face)) + { + nosEngine.LogE("TextRender: could not open font '%s'", fontPath.c_str()); + return false; + } + FT_Set_Pixel_Sizes(face, 0, static_cast(REF_PIXEL_SIZE)); + + RefLineHeight = (face->size->metrics.height >> 6) > 0 ? float(face->size->metrics.height >> 6) + : REF_PIXEL_SIZE * 1.2f; + RefAscender = (face->size->metrics.ascender >> 6) > 0 ? float(face->size->metrics.ascender >> 6) + : REF_PIXEL_SIZE; + + struct Raster + { + std::vector Pixels; + int W = 0, H = 0; + }; + std::array rasters{}; + std::array glyphs{}; + + for (uint32_t i = 0; i < GLYPH_COUNT; ++i) + { + Glyph& g = glyphs[i]; + if (FT_Load_Char(face, FIRST_CHAR + i, FT_LOAD_DEFAULT)) + continue; + FT_GlyphSlot slot = face->glyph; + g.Advance = float(slot->advance.x >> 6); + + if (FT_Render_Glyph(slot, FT_RENDER_MODE_SDF)) + continue; // whitespace / empty outline: advance is still valid + + const FT_Bitmap& bm = slot->bitmap; + if (bm.width == 0 || bm.rows == 0) + continue; + + g.Width = int(bm.width); + g.Height = int(bm.rows); + g.BearingLeft = float(slot->bitmap_left); + g.BearingTop = float(slot->bitmap_top); + g.HasBitmap = true; + + Raster& r = rasters[i]; + r.W = int(bm.width); + r.H = int(bm.rows); + r.Pixels.resize(size_t(r.W) * r.H); + const int pitch = bm.pitch; + for (int row = 0; row < r.H; ++row) + { + const uint8_t* src = bm.buffer + size_t(row) * (pitch < 0 ? -pitch : pitch); + std::memcpy(r.Pixels.data() + size_t(row) * r.W, src, r.W); + } + } + FT_Done_Face(face); + + // Shelf-pack the glyph bitmaps into a fixed-width atlas. + int x = ATLAS_PADDING, y = ATLAS_PADDING, shelfH = 0; + for (uint32_t i = 0; i < GLYPH_COUNT; ++i) + { + Glyph& g = glyphs[i]; + if (!g.HasBitmap) + continue; + if (x + g.Width + ATLAS_PADDING > ATLAS_WIDTH) + { + x = ATLAS_PADDING; + y += shelfH + ATLAS_PADDING; + shelfH = 0; + } + g.AtlasX = x; + g.AtlasY = y; + x += g.Width + ATLAS_PADDING; + shelfH = std::max(shelfH, g.Height); + } + const int atlasW = ATLAS_WIDTH; + const int atlasH = y + shelfH + ATLAS_PADDING; + + std::vector pixels(size_t(atlasW) * atlasH, 0); + for (uint32_t i = 0; i < GLYPH_COUNT; ++i) + { + const Glyph& g = glyphs[i]; + const Raster& r = rasters[i]; + if (!g.HasBitmap) + continue; + for (int row = 0; row < r.H; ++row) + std::memcpy(pixels.data() + size_t(g.AtlasY + row) * atlasW + g.AtlasX, + r.Pixels.data() + size_t(row) * r.W, + r.W); + } + + DestroyAtlas(); + + nosResourceShareInfo atlas{}; + atlas.Info.Type = NOS_RESOURCE_TYPE_TEXTURE; + atlas.Info.Texture = {.Width = uint32_t(atlasW), + .Height = uint32_t(atlasH), + .Format = NOS_FORMAT_R8_UNORM}; + auto cmd = vkss::BeginCmd(NOS_NAME("TextRenderAtlasUpload"), NodeId); + nosResult res = nosVulkan->ImageLoad(cmd, + pixels.data(), + nosVec2u{uint32_t(atlasW), uint32_t(atlasH)}, + NOS_FORMAT_R8_UNORM, + &atlas, + "TextRenderAtlas"); + vkss::EndCmd(cmd, NOS_TRUE, nullptr); + if (res != NOS_RESULT_SUCCESS) + { + nosEngine.LogE("TextRender: failed to upload font atlas"); + return false; + } + + atlas.Info.Texture.Filter = NOS_TEXTURE_FILTER_LINEAR; + AtlasTex = atlas; + AtlasW = atlasW; + AtlasH = atlasH; + Glyphs = glyphs; + AtlasValid = true; + return true; + } + + void EnsureAtlas(const char* fontPin) + { + std::string path = ResolveFontPath(fontPin); + if (AtlasValid && BuiltFontPath && *BuiltFontPath == path) + return; + BuiltFontPath = path; + if (BuildAtlas(path)) + ClearNodeStatusMessages(); + else + SetNodeStatusMessage("Could not load font.", fb::NodeStatusMessageType::FAILURE); + } + + const Glyph* GlyphFor(char c) const + { + auto u = uint32_t(uint8_t(c)); + if (u < FIRST_CHAR || u > LAST_CHAR) + return nullptr; + return &Glyphs[u - FIRST_CHAR]; + } + + // One laid-out glyph: index into Glyphs plus its pen origin on the line. + struct Placed + { + uint32_t GlyphIndex; + float PenX; + int Line; + }; + + // Greedy word-wrap layout in output-pixel space. Honors '\n' and breaks + // words longer than maxWidth character by character. + void LayoutText(const char* text, + float scale, + float maxWidth, + std::vector& out, + std::vector& lineWidths) const + { + const float spaceAdvance = [&] { + const Glyph* sp = GlyphFor(' '); + return sp ? sp->Advance * scale : REF_PIXEL_SIZE * 0.3f * scale; + }(); + + int line = 0; + float penX = 0.0f; + lineWidths.push_back(0.0f); + + auto newLine = [&] { + lineWidths[line] = penX; + ++line; + penX = 0.0f; + lineWidths.push_back(0.0f); + }; + auto placeChar = [&](char c) { + const Glyph* g = GlyphFor(c); + if (!g) + return; + auto idx = uint32_t(uint8_t(c)) - FIRST_CHAR; + if (g->HasBitmap && out.size() < MAX_DRAWN_GLYPHS) + out.push_back({idx, penX, line}); + penX += g->Advance * scale; + }; + + std::string word; + auto wordWidth = [&](const std::string& w) { + float width = 0.0f; + for (char c : w) + if (const Glyph* g = GlyphFor(c)) + width += g->Advance * scale; + return width; + }; + auto flushWord = [&] { + if (word.empty()) + return; + float ww = wordWidth(word); + if (ww > maxWidth) + { + // Word does not fit on any line: hard-break per character. + for (char c : word) + { + const Glyph* g = GlyphFor(c); + float adv = g ? g->Advance * scale : 0.0f; + if (penX > 0.0f && penX + adv > maxWidth) + newLine(); + placeChar(c); + } + } + else + { + if (penX > 0.0f && penX + ww > maxWidth) + newLine(); + for (char c : word) + placeChar(c); + } + word.clear(); + }; + + for (const char* p = text; *p; ++p) + { + char c = *p; + if (c == '\n') + { + flushWord(); + newLine(); + } + else if (c == ' ' || c == '\t') + { + flushWord(); + float adv = (c == '\t') ? spaceAdvance * 4.0f : spaceAdvance; + if (penX > 0.0f) + penX += adv; + } + else + { + word.push_back(c); + } + } + flushWord(); + lineWidths[line] = penX; + } + + // Draws a flat-colored rectangle (the text-box background). + void DrawBox(nosCmd cmd, + const nosResourceShareInfo& tex, + float outW, + float outH, + float x, + float y, + float w, + float h, + nosVec4 boxColor) + { + nosVec2 offset{x / outW, y / outH}; + nosVec2 size{w / outW, h / outH}; + std::array bindings = {vkss::ShaderBinding(NSN_Offset, offset), + vkss::ShaderBinding(NSN_Size, size), + vkss::ShaderBinding(NSN_BoxColor, boxColor)}; + nosVertexData vertexData{ + .DepthFunc = NOS_DEPTH_FUNCTION_ALWAYS, + .DepthWrite = NOS_FALSE, + .DepthTest = NOS_FALSE, + }; + nosRunPassParams pass{.Key = NSN_TextBox_Pass, + .Bindings = bindings.data(), + .BindingCount = uint32_t(bindings.size()), + .Output = tex, + .Vertices = vertexData, + .Wireframe = NOS_FALSE, + .Benchmark = NOS_FALSE, + .DoNotClear = NOS_TRUE}; + nosVulkan->RunPass(cmd, &pass); + } + + // Draws one glyph quad. Used for both the shadow and the fill/stroke pass: + // the shadow passes the shadow color as the fill with a softened edge. + void DrawGlyph(nosCmd cmd, + const nosResourceShareInfo& tex, + float outW, + float outH, + float glyphLeft, + float glyphTop, + float glyphW, + float glyphH, + nosVec4 atlasRect, + nosVec4 fillColor, + nosVec4 strokeColor, + float strokeWidth, + float softness, + float pxRange) + { + nosVec2 offset{glyphLeft / outW, glyphTop / outH}; + nosVec2 size{glyphW / outW, glyphH / outH}; + std::array bindings = {vkss::ShaderBinding(NSN_Offset, offset), + vkss::ShaderBinding(NSN_Size, size), + vkss::ShaderBinding(NSN_AtlasRect, atlasRect), + vkss::ShaderBinding(NSN_FillColor, fillColor), + vkss::ShaderBinding(NSN_StrokeColor, strokeColor), + vkss::ShaderBinding(NSN_StrokeWidth, strokeWidth), + vkss::ShaderBinding(NSN_Softness, softness), + vkss::ShaderBinding(NSN_PxRange, pxRange), + vkss::ShaderBinding(NSN_Atlas, AtlasTex)}; + nosVertexData vertexData{ + .DepthFunc = NOS_DEPTH_FUNCTION_ALWAYS, + .DepthWrite = NOS_FALSE, + .DepthTest = NOS_FALSE, + }; + nosRunPassParams pass{.Key = NSN_TextGlyph_Pass, + .Bindings = bindings.data(), + .BindingCount = uint32_t(bindings.size()), + .Output = tex, + .Vertices = vertexData, + .Wireframe = NOS_FALSE, + .Benchmark = NOS_FALSE, + .DoNotClear = NOS_TRUE}; + nosVulkan->RunPass(cmd, &pass); + } + + nosResult ExecuteNode(nosNodeExecuteParams* rawParams) override + { + NodeExecuteParams args(rawParams); + + const char* fontPin = args.GetPinData(NSN_Font); + EnsureAtlas(fontPin); + + auto resolution = *reinterpret_cast(args[NSN_Resolution].Data->Data); + if (resolution.x == 0 || resolution.y == 0) + return NOS_RESULT_SUCCESS; + + // Resize the output texture to match the requested resolution. + auto tex = vkss::DeserializeTextureInfo(args[NSN_Output].Data->Data); + if (tex.Info.Texture.Width != resolution.x || tex.Info.Texture.Height != resolution.y) + { + auto resized = tex; + resized.Memory = {}; + resized.Info.Texture.Width = resolution.x; + resized.Info.Texture.Height = resolution.y; + auto texFb = vkss::ConvertTextureInfo(resized); + texFb.unscaled = true; + auto buf = nos::Buffer::From(texFb); + nosEngine.SetPinValue(args[NSN_Output].Id, {.Data = buf.Data(), .Size = buf.Size()}); + tex = vkss::DeserializeTextureInfo(args[NSN_Output].Data->Data); + } + if (tex.Memory.Handle == 0) + return NOS_RESULT_SUCCESS; + + const char* text = args.GetPinData(NSN_Text); + float fontSize = *reinterpret_cast(args[NSN_FontSize].Data->Data); + float opacity = std::clamp(*reinterpret_cast(args[NSN_Opacity].Data->Data), 0.0f, 1.0f); + auto textColor = *reinterpret_cast(args[NSN_Color].Data->Data); + auto strokeColor = *reinterpret_cast(args[NSN_StrokeColor].Data->Data); + float strokeWidthPin = *reinterpret_cast(args[NSN_StrokeWidth].Data->Data); + auto shadowColor = *reinterpret_cast(args[NSN_ShadowColor].Data->Data); + auto shadowOffset = *reinterpret_cast(args[NSN_ShadowOffset].Data->Data); + float shadowSoftnessPin = *reinterpret_cast(args[NSN_ShadowSoftness].Data->Data); + auto boxColor = *reinterpret_cast(args[NSN_BackgroundColor].Data->Data); + auto boxPadding = *reinterpret_cast(args[NSN_BackgroundPadding].Data->Data); + auto hAlign = *reinterpret_cast(args[NSN_HorizontalAlign].Data->Data); + auto vAlign = *reinterpret_cast(args[NSN_VerticalAlign].Data->Data); + auto position = *reinterpret_cast(args[NSN_Position].Data->Data); + auto wrapWidthChars = *reinterpret_cast(args[NSN_WrapWidth].Data->Data); + + // Global opacity folds into every color's alpha. + textColor.w *= opacity; + strokeColor.w *= opacity; + shadowColor.w *= opacity; + boxColor.w *= opacity; + + // The frame outside the text box stays transparent. + auto cmd = vkss::BeginCmd(NOS_NAME("TextRender"), NodeId); + nosVulkan->Clear(cmd, &tex, nosVec4{0.0f, 0.0f, 0.0f, 0.0f}); + + if (AtlasValid && text && text[0] != '\0' && fontSize > 0.0f) + { + const float outW = float(tex.Info.Texture.Width); + const float outH = float(tex.Info.Texture.Height); + const float scale = fontSize / REF_PIXEL_SIZE; + const float lineHeight = RefLineHeight * scale; + const float ascender = RefAscender * scale; + const float pxRange = 2.0f * float(SDF_SPREAD) * scale; + // The SDF only carries data within SDF_SPREAD reference pixels of the + // glyph edge, which bounds outline thickness and shadow softness. + const float effectLimit = float(SDF_SPREAD) * scale; + const float strokeWidth = std::clamp(strokeWidthPin, 0.0f, effectLimit); + const float shadowSoftness = std::clamp(shadowSoftnessPin, 0.0f, effectLimit); + + // WrapWidth is in characters; 0 falls back to the texture width. + // The character cell width is the font's space advance, which is + // exact for monospace fonts and approximate for proportional ones. + float wrapWidth = outW; + if (wrapWidthChars > 0) + { + const Glyph* space = GlyphFor(' '); + const float charWidth = (space ? space->Advance : REF_PIXEL_SIZE * 0.6f) * scale; + wrapWidth = float(wrapWidthChars) * charWidth; + } + + std::vector placed; + std::vector lineWidths; + LayoutText(text, scale, wrapWidth, placed, lineWidths); + + const uint32_t numLines = uint32_t(lineWidths.size()); + const float blockHeight = lineHeight * float(numLines); + + float vBase = 0.0f; + if (vAlign == 1) // MIDDLE + vBase = (outH - blockHeight) * 0.5f; + else if (vAlign == 2) // BOTTOM + vBase = outH - blockHeight; + const float vOffset = vBase + position.y; + + // Per-line horizontal anchor offset (alignment + position nudge). + std::vector hOffsets(numLines); + for (uint32_t i = 0; i < numLines; ++i) + { + float off = 0.0f; + if (hAlign == 1) // CENTER + off = (outW - lineWidths[i]) * 0.5f; + else if (hAlign == 2) // RIGHT + off = outW - lineWidths[i]; + hOffsets[i] = off + position.x; + } + + // Text-block bounds, used for the background box. + float blockLeft = outW, blockRight = 0.0f; + bool anyLine = false; + for (uint32_t i = 0; i < numLines; ++i) + { + if (lineWidths[i] <= 0.0f) + continue; + anyLine = true; + blockLeft = std::min(blockLeft, hOffsets[i]); + blockRight = std::max(blockRight, hOffsets[i] + lineWidths[i]); + } + + // Background box, behind everything. + if (anyLine && boxColor.w > 0.0f) + DrawBox(cmd, + tex, + outW, + outH, + blockLeft - boxPadding.x, + vOffset - boxPadding.y, + (blockRight - blockLeft) + 2.0f * boxPadding.x, + blockHeight + 2.0f * boxPadding.y, + boxColor); + + auto glyphRect = [&](const Placed& gp, float& left, float& top, float& w, float& h) { + const Glyph& g = Glyphs[gp.GlyphIndex]; + const float baseline = vOffset + ascender + float(gp.Line) * lineHeight; + left = hOffsets[gp.Line] + gp.PenX + g.BearingLeft * scale; + top = baseline - g.BearingTop * scale; + w = float(g.Width) * scale; + h = float(g.Height) * scale; + }; + auto atlasRectOf = [&](const Placed& gp) { + const Glyph& g = Glyphs[gp.GlyphIndex]; + return nosVec4{float(g.AtlasX) / float(AtlasW), + float(g.AtlasY) / float(AtlasH), + float(g.Width) / float(AtlasW), + float(g.Height) / float(AtlasH)}; + }; + + // Drop shadow: all shadows first so no glyph fill is tinted by a + // neighbouring glyph's shadow. + if (shadowColor.w > 0.0f) + { + nosVec4 noStroke{0.0f, 0.0f, 0.0f, 0.0f}; + for (const Placed& gp : placed) + { + float left, top, w, h; + glyphRect(gp, left, top, w, h); + DrawGlyph(cmd, + tex, + outW, + outH, + left + shadowOffset.x, + top + shadowOffset.y, + w, + h, + atlasRectOf(gp), + shadowColor, + noStroke, + 0.0f, + shadowSoftness, + pxRange); + } + } + + // Fill + outline. + for (const Placed& gp : placed) + { + float left, top, w, h; + glyphRect(gp, left, top, w, h); + DrawGlyph(cmd, + tex, + outW, + outH, + left, + top, + w, + h, + atlasRectOf(gp), + textColor, + strokeColor, + strokeWidth, + 0.0f, + pxRange); + } + } + + vkss::EndCmd(cmd, NOS_FALSE, nullptr); + return NOS_RESULT_SUCCESS; + } +}; + +static nosResult RegisterShaderPair(const fs::path& root, + const char* baseName, + nosName fragKey, + nosName vertKey) +{ + auto fragPath = (root / "Shaders" / (std::string(baseName) + ".frag")).generic_string(); + auto vertPath = (root / "Shaders" / (std::string(baseName) + ".vert")).generic_string(); + std::array shaders = { + nosShaderInfo{.ShaderName = fragKey, + .Source = {.Stage = NOS_SHADER_STAGE_FRAG, .GLSLPath = fragPath.c_str()}, + .AssociatedNodeClassName = NSN_TextRender}, + nosShaderInfo{.ShaderName = vertKey, + .Source = {.Stage = NOS_SHADER_STAGE_VERT, .GLSLPath = vertPath.c_str()}, + .AssociatedNodeClassName = NSN_TextRender}, + }; + return nosVulkan->RegisterShaders(shaders.size(), shaders.data()); +} + +nosResult RegisterTextRender(nosNodeFunctions* fn) +{ + NOS_BIND_NODE_CLASS(NSN_TextRender, TextRenderNode, fn); + + fs::path root = nosEngine.Module->RootFolderPath; + if (nosResult ret = RegisterShaderPair(root, "TextGlyph", NSN_TextGlyph_Frag, NSN_TextGlyph_Vert); + ret != NOS_RESULT_SUCCESS) + return ret; + if (nosResult ret = RegisterShaderPair(root, "TextBox", NSN_TextBox_Frag, NSN_TextBox_Vert); + ret != NOS_RESULT_SUCCESS) + return ret; + + std::array passes = { + nosPassInfo{ + .Key = NSN_TextGlyph_Pass, + .Shader = NSN_TextGlyph_Frag, + .VertexShader = NSN_TextGlyph_Vert, + .MultiSample = 1, + .Blend = NOS_BLEND_MODE_ALPHA_BLENDING, + }, + nosPassInfo{ + .Key = NSN_TextBox_Pass, + .Shader = NSN_TextBox_Frag, + .VertexShader = NSN_TextBox_Vert, + .MultiSample = 1, + .Blend = NOS_BLEND_MODE_ALPHA_BLENDING, + }, + }; + return nosVulkan->RegisterPasses(passes.size(), passes.data()); +} + +} // namespace nos::utilities diff --git a/Plugins/nosUtilities/Source/UtilitiesMain.cpp b/Plugins/nosUtilities/Source/UtilitiesMain.cpp index c3d3e24a..ad07f07c 100644 --- a/Plugins/nosUtilities/Source/UtilitiesMain.cpp +++ b/Plugins/nosUtilities/Source/UtilitiesMain.cpp @@ -41,7 +41,9 @@ enum Utilities : int PropagateExecution, UploadBufferProvider, BoundedQueue, + MultiBoundedQueue, RingBuffer, + MultiRingBuffer, Host, DeinterlacedBoundedTextureQueue, DeinterlacedBufferRing, @@ -57,6 +59,9 @@ enum Utilities : int GridOutputLayout, LoadCubeLUT, RepeatingJunction, + MultiLiveOut, + TextRender, + ScheduleRequest, Count }; @@ -75,7 +80,9 @@ nosResult RegisterSink(nosNodeFunctions*); nosResult RegisterPropagateExecution(nosNodeFunctions*); nosResult RegisterUploadBufferProvider(nosNodeFunctions*); nosResult RegisterBoundedQueue(nosNodeFunctions*); +nosResult RegisterMultiBoundedQueue(nosNodeFunctions*); nosResult RegisterRingBuffer(nosNodeFunctions*); +nosResult RegisterMultiRingBuffer(nosNodeFunctions*); nosResult RegisterHost(nosNodeFunctions*); nosResult RegisterPin2Json(nosNodeFunctions*); nosResult RegisterJson2Pin(nosNodeFunctions*); @@ -93,6 +100,9 @@ nosResult RegisterFreeOutputLayout(nosNodeFunctions*); nosResult RegisterGridOutputLayout(nosNodeFunctions*); nosResult RegisterLoadCubeLUT(nosNodeFunctions*); nosResult RegisterRepeatingJunction(nosNodeFunctions*); +nosResult RegisterMultiLiveOut(nosNodeFunctions*); +nosResult RegisterTextRender(nosNodeFunctions*); +nosResult RegisterScheduleRequest(nosNodeFunctions*); nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** outList) { @@ -129,7 +139,9 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** ou GEN_CASE_NODE(PropagateExecution) GEN_CASE_NODE(UploadBufferProvider) GEN_CASE_NODE(BoundedQueue) + GEN_CASE_NODE(MultiBoundedQueue) GEN_CASE_NODE(RingBuffer) + GEN_CASE_NODE(MultiRingBuffer) GEN_CASE_NODE(Host) GEN_CASE_NODE(DeinterlacedBoundedTextureQueue) GEN_CASE_NODE(DeinterlacedBufferRing) @@ -145,6 +157,9 @@ nosResult NOSAPI_CALL ExportNodeFunctions(size_t* outSize, nosNodeFunctions** ou GEN_CASE_NODE(GridOutputLayout) GEN_CASE_NODE(LoadCubeLUT) GEN_CASE_NODE(RepeatingJunction) + GEN_CASE_NODE(MultiLiveOut) + GEN_CASE_NODE(TextRender) + GEN_CASE_NODE(ScheduleRequest) } } return NOS_RESULT_SUCCESS; @@ -163,7 +178,7 @@ NOSAPI_ATTR nosResult NOSAPI_CALL nosExportPlugin(nosPluginFunctions* out) } // clang-format off outRenamedFrom[0] = NOS_NAME("nos.fb.ChannelViewerChannels"); outRenamedTo[0] = NOS_NAME("nos.utilities.ChannelViewerChannels"); - outRenamedFrom[1] = NOS_NAME("nos.fb.ChannelViewerFormats"); outRenamedTo[1] = NOS_NAME("nos.utilities.ChannelViewerFormats"); + outRenamedFrom[1] = NOS_NAME("nos.fb.ChannelViewerFormats"); outRenamedTo[1] = NOS_NAME("nos.mediaio.ColorSpace"); outRenamedFrom[2] = NOS_NAME("nos.fb.GradientKind"); outRenamedTo[2] = NOS_NAME("nos.utilities.GradientKind"); outRenamedFrom[3] = NOS_NAME("nos.fb.BlendMode"); outRenamedTo[3] = NOS_NAME("nos.utilities.BlendMode"); outRenamedFrom[4] = NOS_NAME("nos.fb.ResizeMethod"); outRenamedTo[4] = NOS_NAME("nos.utilities.ResizeMethod"); diff --git a/Plugins/nosUtilities/Utilities.noscfg b/Plugins/nosUtilities/Utilities.noscfg index 79883ae9..50864c54 100644 --- a/Plugins/nosUtilities/Utilities.noscfg +++ b/Plugins/nosUtilities/Utilities.noscfg @@ -2,7 +2,7 @@ "info": { "id": { "name": "nos.utilities", - "version": "3.14.8" + "version": "3.15.0" }, "description": "Various utility nodes.", "display_name": "Utilities", @@ -43,7 +43,9 @@ "Config/UploadBufferProvider.nosdef", "Config/TimedFunctionSignaller.nosdef", "Config/RingBuffer.nosdef", + "Config/MultiRingBuffer.nosdef", "Config/BoundedQueue.nosdef", + "Config/MultiBoundedQueue.nosdef", "Config/Host.nosdef", "Config/AutoResize.nosdef", "Config/ExecDepend.nosdef", @@ -63,7 +65,10 @@ "Config/CalculateDispatchSize.nosdef", "Config/YADIF.nosdef", "Config/YADIFWithAutoDispatchSize.nosdef", - "Config/RepeatingJunction.nosdef" + "Config/RepeatingJunction.nosdef", + "Config/MultiLiveOut.nosdef", + "Config/TextRender.nosdef", + "Config/ScheduleRequest.nosdef" ], "custom_types": [ "Config/Merge.fbs", @@ -73,7 +78,8 @@ "Config/TextureSwitcher.fbs", "Config/ChannelViewer.fbs", "Config/Sink.fbs", - "Config/Layout.fbs" + "Config/Layout.fbs", + "Config/TextRender.fbs" ], "defaults": [ "Config/Defaults.json" diff --git a/Subsystems/nosTrackSubsystem/Config/Track.fbs b/Subsystems/nosTrackSubsystem/Config/Track.fbs index e1dcce23..f36f2818 100644 --- a/Subsystems/nosTrackSubsystem/Config/Track.fbs +++ b/Subsystems/nosTrackSubsystem/Config/Track.fbs @@ -42,3 +42,18 @@ enum RotationSystem : uint { RPT = 4, PRT = 5, } + +// World coordinate frame convention used by a Track endpoint. Encodes axis +// assignments to world-semantic directions (forward, right, up), the implied +// handedness, and the Euler convention for the Track.rotation field. +enum CoordinateFrame : ubyte { + // Left-handed, Z-up. +X forward, +Y right, +Z up. + // Rotation: rot.x = roll (X), rot.y = pitch (Y), rot.z = yaw (Z), + // intrinsic ZYX => R = Rz(yaw) * Ry(pitch) * Rx(roll). + LH_ZUp_FwdX_RightY = 0, + + // Right-handed, Y-up. +X right, +Y up, -Z forward. + // Rotation: rot.x = pitch (X), rot.y = yaw (Y), rot.z = roll (Z), + // intrinsic YXZ => R = Ry(yaw) * Rx(pitch) * Rz(roll). + RH_YUp_FwdNegZ_RightX = 1, +} diff --git a/Subsystems/nosTrackSubsystem/nosTrackSubsystem.nossys b/Subsystems/nosTrackSubsystem/nosTrackSubsystem.nossys index d2f6b9cb..625fd3fe 100644 --- a/Subsystems/nosTrackSubsystem/nosTrackSubsystem.nossys +++ b/Subsystems/nosTrackSubsystem/nosTrackSubsystem.nossys @@ -2,7 +2,7 @@ "info": { "id": { "name": "nos.sys.track", - "version": "1.0.0" + "version": "1.1.0" }, "display_name": "Track Subsystem", "dependencies": [