diff --git a/source/_static/custom.css b/source/_static/custom.css index f855117..5cd11c2 100644 --- a/source/_static/custom.css +++ b/source/_static/custom.css @@ -11,3 +11,10 @@ html[data-theme="light"] { html[data-theme="dark"] { --pst-color-primary: #8800ff; } + + +/* Ensure the SVG diagrams scale with font size, not diagram width */ +img[src$=".svg"] { + max-width: 100%; + height: auto; +} diff --git a/source/conf.py b/source/conf.py index ce22855..07c8fe2 100644 --- a/source/conf.py +++ b/source/conf.py @@ -35,6 +35,8 @@ # ones. extensions = [ "sphinxcontrib.plantuml", + "sphinx.ext.mathjax", + "sphinx.ext.todo", ] # Add any paths that contain templates here, relative to this directory. @@ -70,11 +72,8 @@ version_match = "v" + release html_theme_options = { - # "logo_link": "", "github_url": "https://github.com/vodf/vodf-docs", "icon_links_label": "Quick Links", - # "show_nav_level": 4, - # "navigation_depth": 4, "use_edit_page_button": True, "navbar_align": "content", "navbar_end": ["version-switcher", "navbar-icon-links"], # "theme-switcher" @@ -84,9 +83,16 @@ }, "header_links_before_dropdown": 6, "announcement": "

This is an unreleased version, informaton may not be correct

", - "footer_start": ["copyright",], + "footer_start": [ + "copyright", + ], "footer_center": ["last-updated"], "footer_end": ["sphinx-version", "theme-version"], + "logo": { + "alt_text":"VODF: Very-High-Energy Open Data Format", + "image_light": "source/_static/VODF-logo.png", + "image_dark": "source/_static/VODF-logo.png", + }, } html_context = { @@ -101,7 +107,6 @@ html_css_files = [ "custom.css", ] -html_logo = "_static/VODF-logo.svg" # Add any paths that contain custom static files (such as style sheets) here, @@ -109,6 +114,8 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] +# Show TODO entries. Disable this before release +todo_include_todos = True # ====================================================================== # Options for LaTeX docment output diff --git a/source/specification/common.inc b/source/specification/common.inc new file mode 100644 index 0000000..56f896a --- /dev/null +++ b/source/specification/common.inc @@ -0,0 +1,43 @@ +@startuml +hide circles +hide methods +skinparam DefaultFontSize 14 +skinparam DefaultFontName "Arial" + + + + + +skinparam classStereotypeFontColor white + +hide <> stereotype + + +@enduml diff --git a/source/specification/core-concepts.rst b/source/specification/core-concepts.rst index 473c4f0..976f3f9 100644 --- a/source/specification/core-concepts.rst +++ b/source/specification/core-concepts.rst @@ -3,3 +3,47 @@ Core Concepts This describes the overall data model of VODF and how its component parts are inter-related. + +Data Levels +----------- + +.. todo:: + + Describe high-level relationship between L1 to L4 + +Common Types +------------ + +.. uml:: + + !include common.inc + class TimeStamp + +Maps and N-Dimensional Arrays +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + + + +PDGID +~~~~~ + +.. uml:: + + !include common.inc + class PDGID <> + + +Particle ids used in VODF shall follow the integer `https://particle.wiki/wiki/PDG_particle_numbering_scheme `_, e.g.: + + +.. csv-table:: Some example PDGIDs + :header: "Particle","ID" + :widths: 1,5 + + ":math:`\gamma`", 22 + ":math:`e-`", 11 + ":math:`p`", 2212 + ":math:`\nu_e`", 12 + ":math:`\nu_\mu`", 14 + ":math:`\nu_\tau`",16 diff --git a/source/specification/level-1/coverage.plantuml b/source/specification/level-1/coverage.plantuml new file mode 100644 index 0000000..29f2973 --- /dev/null +++ b/source/specification/level-1/coverage.plantuml @@ -0,0 +1,46 @@ +@startuml +!include ../common.inc +skinparam linetype ortho + +package IVOA <> { + + class SMOC + class TMOC + class STMOC +} + +package VODF { + package L1 { + class SpatialCoverage { + + r_fov : float + ..optional.. + + region : Region + } + + class TemporalCoverage { + + t_start + + t_stop + } + + class SpectralCoverage { + + energy_min + + energy_max + } + + Coverage *-- SpatialCoverage + Coverage *-- TemporalCoverage + Coverage *-- SpectralCoverage + TemporalCoverage *-d--- "0..1" IVOA.TMOC + TemporalCoverage *-d--- "0..1" IVOA.STMOC + SpatialCoverage *-d--- "0..1" IVOA.SMOC + SpatialCoverage *-d--- "0..1" IVOA.STMOC + + + class OnTime <> + class Pointing <> + TemporalCoverage ..> OnTime + TemporalCoverage ..> Pointing + } +} + +@enduml diff --git a/source/specification/level-1/eventlist.plantuml b/source/specification/level-1/eventlist.plantuml new file mode 100644 index 0000000..ae79ac3 --- /dev/null +++ b/source/specification/level-1/eventlist.plantuml @@ -0,0 +1,38 @@ +@startuml +!include ../common.inc + + +package VODF.L1 { + + class EventList { + + collection_id + + obs_id + } + class Event { + + event_id: int64 + -- + + ra_reco: float32 + + dec_reco: float32 + + time_reco: TimeStamp + + energy_reco: float32 + ..optional.. + + event_type: int + + alt_reco: float32 + + az_reco: float32 + } + + class SimulatedEvent { + + alt_true: float32 + + az_true: float32 + + energy_true: float32 + + time_true: TimeStamp + + particle_id_true : PDGID + ..optional.. + + ra_true: float32 + + dec_true: float32 + } + + EventList "1" o-- "0..*" Event + Event -r-|> SimulatedEvent +} +@enduml diff --git a/source/specification/level-1/index.rst b/source/specification/level-1/index.rst index 6966845..1e5ed90 100644 --- a/source/specification/level-1/index.rst +++ b/source/specification/level-1/index.rst @@ -3,18 +3,18 @@ ========= VODF Level-1 contains data products related to specific -:term:`observations`, and in particular to :term:`events` detected +:term:`observations `, and in particular to :term:`events` detected by a given instrument as well as :term:`IRF` that describe how the parameters of those events map to physical quantities. This data level may contain extra information on the instrument pointing, provenance information, :term:`stable time intervals or instrumental good time intervals`, systematics error or quality estimation of the :term:`IRF`. It also covers related information such as time-series tables of data quality, instrumental or atmospheric conditions. -Level-1 data is assumed to be already *pre-processed* by the instrument that -produces it. The pre-processing must include all calibration and +Level-1 data products are assumed to be already *pre-processed* by the instrument that +produces them. The pre-processing must include all calibration and :term:`reconstruction` necessary to reduce the raw data of the instrument into a set of physical estimated parameters per detected particle. -.. note:: VODF Level-1 is equivalent to *data level 3 (DL3)* defined by CTAO, +.. tip:: VODF Level-1 is equivalent to *data level 3 (DL3)* defined by CTAO, also called "science-ready data". In CTAO, levels DL0-DL2 constitute the *raw* and *pre-processed* data levels that are out of scope for VODF. @@ -22,10 +22,8 @@ set of physical estimated parameters per detected particle. Data Model ========== - .. uml:: level-1.plantuml :caption: VODF **Level-1** Data Model - :width: 600px Observations ------------ @@ -41,7 +39,7 @@ are not purely hardware, but a combination of hardware and software, and the details of the *low-level data processing changes the characteristics of the instrument*. With improvements to this low-level processing, the sensitivity, angular resolution, energy resolution, and other factors can change. A second -consequence of :term:`observations` being software-defined is that the +consequence of :term:`observations ` being software-defined is that the minimal duration is therefore up to the observatory. It can be directly related to one *data acquisition interval*, but can also be a subset or concatenation of many. @@ -56,5 +54,131 @@ Data Releases TBD +Coverage +-------- + +.. uml:: coverage.plantuml + :caption: VODF **Level-1 Coverage** Data Model + +EventList +---------- + +.. uml:: eventlist.plantuml + :caption: VODF **Level-1 EventList** Data Model + +IRF +--- + +.. uml:: irf.plantuml + :caption: VODF **Level-1 IRF** Data Model + +The :term:`IRF` contains the information necessary to map instrumental +:term:`reconstructed ` parameters of the event to *physical* +parameters, i.e. it allows one to transform from a physical *flux* in a given +space-time-spectral interval into a predicted number of detected *counts* for a +given instrument. The IRF is often decomposed into a set of independent +``IRFComponents``, e.g. by separating spatial and spectral aspects. An IRF may +also vary in time, due to changing observation conditions, or instrumental +degradation. The assumption in VODF is that time variations are ignored and +therefore an :term:`observation` is broken into pieces where the IRF remains +stable, and residual variations are handled by adding instrumental parameters. + +.. tip:: + + Science analysis with event-counting instruments often involves fitting a + physical model to observed data. The standard technique is a *maximum + likelihood fit* using *forward folding*, where the physical model is passed + through the IRF to transform flux (physical units) into predicted counts + (instrumental units), and the fit is performed on this transformed quantity + by comparing the predicted to measured counts. The opposite process, + *unfolding*, where instrumental uncertainties are removed by deconvolution + and the model is fit in physical units gives unstable results when data or + IRFs contain instrumental or statistical noise. + + +StandardIRF +~~~~~~~~~~~ + +For VODF, we provide a commonly used decomposition of the :term:`IRF` that +separates the response :math:`\hat R` from the background rate :math:`B`. This +is convenient since :math:`\hat R` is usually computed from simulations, while +:math:`B` can be computed from observed data using blank fields. Then, the +predicted instrumental counts :math:`N` can be computed for a given flux +:math:`F` as: + +.. math:: + + \underbrace{N(\vec{p}', E' | \vec{p}, E)}_{Instrumental\ Counts} = \int d\Omega dE \; \hat{R}(\vec{p}',E'|\vec{p},E) \, \underbrace{F(\vec{p}, E)}_{Source\ Model} + \int d\Omega dE \; B(\vec{p}', E') + +where :math:`(\vec{p}, E)` are the *true* primary particle point of origin and +its energy, and :math:`(\vec{p}',E')` are the corresponding :term:`reconstructed +` quantities. The coordinate system for :math:`\vec p` is +usually represented in polar coordinates relative to the FOV center, though +other representations are possible. Then, we decompose :math:`R` into the +following components: + +.. math:: + + R(E', {\vec{p'}} | E, \vec{p}, t) = + \underbrace{A_\text{eff}(E, \vec{p}, t)}_{{\text{Effective Area}}} + \cdot \overbrace{M(E' | E, \vec{p}, t)}^{{\text{Energy Migration}}} + \cdot \underbrace{P(\vec{p'} | E, \vec{p}, t)}_{{\text{Point Spread Function}}}. + + +Effective Collection Area Function (:math:`A_\mathrm{eff}`, ``EffectiveArea``) + Given a set of physical parameters, provides the collection area of the + instrument, computed usually from detailed simulations as the ratio of the + number of detected events to the total simulated multiplied by the area over + which they were simulated, for given true position in the field of view and + true energy. + +Point-Spread Function (:math:`P`, ``PSF``) + The probability to reconstruct an event a point :math:`\vec p'` in the FOV + if it had a true position :math:`\vec p`. Positional bias is ignored, + therefore it represents only the dispersion, i.e. the mean reconstructed + position is assumed equal to the true position. + +Energy Migration Function (:math:`M`, ``EnergyMigration``) + The probability to reconstruct the energy as :math:`E'` for a given true + energy :math:`E`, including both dispersion and bias. This sometimes called + the *energy redistribution matrix*, or the *redistribution matrix file + (RMF)*. + +Background Rate Function (:math:`B`, ``BackgroundRate``) + The expected rate of background events (in counts/second) at a given point + :math:`\vec p` and energy `E`. Often, the real rate is difficult to compute + correctly without real data due to e.g. atmospheric uncertainties, and + therefore it is important to note that it may need to be calibrated using + real data. I.e. the shape should be correct, but the normalization may need + to be refined. + +.. note:: + + This decomposition ignores cross-terms, like the correlation between spectral + and spatial resolution, which can be important in some cases. Future versions + of VODF may include more detailed decompositions. + + +.. todo:: + + Discuss event types here? + + +OnTime +------ + +.. uml:: ontime.plantuml + :caption: VODF **Level-1 OnTime** Data Model + + +Pointing +-------- + +.. uml:: pointing.plantuml + :caption: VODF **Level-1 Pointing** Data Model + + + + Data Products ============= diff --git a/source/specification/level-1/irf.plantuml b/source/specification/level-1/irf.plantuml new file mode 100644 index 0000000..69f24ac --- /dev/null +++ b/source/specification/level-1/irf.plantuml @@ -0,0 +1,21 @@ +@startuml +!include ../common.inc + +package VODF.L1 { + + abstract class IRF + abstract class IRFComponent + class EffectiveArea <> + class BackgroundRate <> + class EnergyMigration <> + class PSF <> + + IRF *-d- "0..*" IRFComponent + IRF <|-r---- StandardIRF + + StandardIRF *--- EffectiveArea + StandardIRF *--- BackgroundRate + StandardIRF *--- EnergyMigration + StandardIRF *--- PSF + +} diff --git a/source/specification/level-1/level-1.plantuml b/source/specification/level-1/level-1.plantuml index 33b65e5..5c375fb 100644 --- a/source/specification/level-1/level-1.plantuml +++ b/source/specification/level-1/level-1.plantuml @@ -1,60 +1,66 @@ @startuml -hide circles -hide methods -skinparam ClassBackgroundColor white/aliceblue -skinparam ClassHeaderBackgroundColor lightblue -skinparam backgroundcolor transparent - - +!include ../common.inc +skinparam linetype ortho -package IVOA { - class ObsCore { +package IVOA <> { + class ObsCore <> {} + class DataLink <> {} + ObsCore -- DataLink +} - } +package VODF { + class Facility + class Instrument + Facility -- Instrument : <> > } -package VODF.L1 { - class DataRelease { - + data_release_id - + time_interval - + facility - + processing_configuration - + processing_pipeline_version - } +package VODF.L1 { - class Observation #line.bold { - + obs_id - + time_interval - + spatial_region : SkyRegion - + instrument_name - + ... + together { + class DataRelease { + + facility: Facility + + data_release_id + .. + ~ time_interval + ~ documentation_url + ..facility-specific properties.. + ~ processing_configuration + ~ processing_pipeline_version + ~ ... + } + + class Observation #line.bold { + + obs_id + -- + ~ coverage: Coverage + ~ primary_particle_id: PDGID + ~ ... + } + + class ObservationCollection { + + } } - Observation "1" -- "0..1" Pointing + ObsCore .r. Observation : "<> >" + DataLink .r. Observation : "<> >" - Pointing <|-- NominalPointing - Pointing <|-- TimeSeriesPointing - - DataRelease "1..*" -- "1" ObservationCollection + DataRelease "1" -- "1..*" ObservationCollection ObservationCollection "1" -- "1..*" Observation - ObsCore -r- Observation : "maps to <" + ' Insrument is contextual info, not a part, so only an association. + Observation "1" --l-- "1" Instrument : used > + Facility "1" -r-- "*" DataRelease + - Observation "1" -- "0..1" EventList - Observation "0..*" *-- "0..1" IRF - Observation "1" -- "1..*" OnTime - OnTime <|-- SingleOnTime - OnTime <|-- TimeSeriesOnTime - EventList "1" -- "0..*" Event + ' Constituent parts of an observation + Observation "1" o-- "1" Pointing + Observation "1" o-- "1..*" OnTime - Event "properties" -- "dimensions" IRFComponent + Observation "1" o-- "0..1" EventList + Observation "0..*" o-- "0..1" IRF IRF "1" *-- "1..*" IRFComponent diff --git a/source/specification/level-1/ontime.plantuml b/source/specification/level-1/ontime.plantuml new file mode 100644 index 0000000..f425979 --- /dev/null +++ b/source/specification/level-1/ontime.plantuml @@ -0,0 +1,17 @@ +@startuml +!include ../common.inc + + +package VODF.L1 { + + abstract class OnTime + + class SingleOnTime + class TimeSeriesOnTime + + OnTime <|-- SingleOnTime + OnTime <|-- TimeSeriesOnTime + + TimeSeriesOnTime o-- "1..*" SingleOnTime +} +@enduml diff --git a/source/specification/level-1/pointing.plantuml b/source/specification/level-1/pointing.plantuml new file mode 100644 index 0000000..2db61f8 --- /dev/null +++ b/source/specification/level-1/pointing.plantuml @@ -0,0 +1,20 @@ +@startuml +!include ../common.inc + + +package VODF.L1 { + abstract class Pointing + + class SinglePointing + class TimeSeriesPointing + + Pointing <|-- SinglePointing + + Pointing <|-- TimeSeriesPointing + TimeSeriesPointing "1" o-- "1..*" SinglePointing + + SinglePointing <|-- HorizontalSinglePointing + SinglePointing <|-- CelestialSinglePointing + +} +@enduml diff --git a/source/specification/level-2/index.rst b/source/specification/level-2/index.rst index 41edaa5..89df969 100644 --- a/source/specification/level-2/index.rst +++ b/source/specification/level-2/index.rst @@ -12,4 +12,3 @@ e.g. counts. .. uml:: level-2.plantuml :caption: VODF **Level-2** Data Model - :width: 600px diff --git a/source/specification/level-2/level-2.plantuml b/source/specification/level-2/level-2.plantuml index 3382c5c..9ab757d 100644 --- a/source/specification/level-2/level-2.plantuml +++ b/source/specification/level-2/level-2.plantuml @@ -1,15 +1,5 @@ @startuml -hide circles -hide methods -skinparam ClassBackgroundColor white/aliceblue -skinparam ClassHeaderBackgroundColor lightblue - - +!include ../common.inc VODF.L1.Observation "1..*" -- "1" VODF.L2.DataSetCollection