From 008b2c519e9aa077c00392c870e24f71e89ba3e5 Mon Sep 17 00:00:00 2001 From: acorrao <95718914+AdamCorrao@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:44:15 -0500 Subject: [PATCH 01/18] build(docs): move the docs to pydata-sphinx-theme with diagrams and tabs Add sphinx-design (cards, tabs, badges), sphinxcontrib-mermaid (diagrams written as text) and sphinx-copybutton to the dev feature; drop sphinx_rtd_theme. conf.py: the new theme, MyST-only sources, mermaid fences rendered as diagrams, and comments without internal tracking IDs. Assisted-by: claude-code:claude-opus-5-5 --- docs/conf.py | 90 +++++++++++++++----------- docs/requirements.txt | 9 ++- pixi.lock | 145 +++++++++++++++++++++++++++++++++--------- pixi.toml | 5 +- 4 files changed, 178 insertions(+), 71 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 6710e86..0251840 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,7 +10,6 @@ sys.path.insert(0, str(Path(__file__).parent.parent / 'src')) # -- Project information ----------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information project = 'PowderLine' copyright = '2026, NSLS-II / Brookhaven National Laboratory' @@ -20,19 +19,21 @@ from powderline import __version__ as release # noqa: E402 # -- General configuration --------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ - 'sphinx.ext.autodoc', # Auto-generate docs from docstrings - 'sphinx.ext.napoleon', # Support Google/NumPy docstring styles - 'sphinx.ext.viewcode', # Add links to source code - 'sphinx.ext.intersphinx', # Link to other project docs (e.g., Python, NumPy) - 'myst_parser', # Support Markdown files + 'sphinx.ext.autodoc', # API pages from docstrings + 'sphinx.ext.napoleon', # Google/NumPy docstring styles + 'sphinx.ext.viewcode', # links to highlighted source + 'sphinx.ext.intersphinx', # links to Python, NumPy, pandas, pydantic docs + 'myst_parser', # Markdown (MyST) pages + 'sphinx_design', # cards, grids, tabs, badges + 'sphinxcontrib.mermaid', # diagrams written as text + 'sphinx_copybutton', # a copy button on code blocks ] -# GSAS-II is a heavy conda dependency absent from the doc-build -# environment; mock it so autodoc can import the GSAS-II gateway's runtime -# modules (gateways/gsasii/{setters,extractors,...}.py import GSASII at top). +# GSAS-II is a heavy conda dependency that autodoc does not need: mock it so +# the GSAS-II gateway's runtime modules (which import GSASII at module level) +# can be imported for their docstrings. autodoc_mock_imports = ["GSASII"] # Napoleon settings for Google/NumPy style docstrings @@ -48,7 +49,6 @@ napoleon_use_param = True napoleon_use_rtype = True -# Intersphinx mapping to link to external docs intersphinx_mapping = { 'python': ('https://docs.python.org/3', None), 'numpy': ('https://numpy.org/doc/stable/', None), @@ -57,13 +57,16 @@ } templates_path = ['_templates'] -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'dev', # dev/ = in-repo development dossiers, not user-facing docs - '_generated'] # script-generated fragments pulled in with MyST {include}, not pages of their own +exclude_patterns = [ + '_build', 'Thumbs.db', '.DS_Store', + '_generated', # script-generated fragments pulled in with MyST {include}, not pages of their own + 'catalog', # the public engine-behaviour / known-issues catalogue (data, rendered by a script) +] # Treat missing cross-reference targets as warnings (not silently ignored) nitpicky = True -# cases without a link target +# Cross-references that have no link target nitpick_ignore = [ # pandas' public docs index `pandas.DataFrame`, but runtime type hints # resolve to its internal module path; the pandas intersphinx inventory @@ -74,17 +77,16 @@ # inventory to link against. ('py:class', 'annotated_types.Gt'), ('py:class', 'annotated_types.Ge'), - # The GSAS-II extractors' signatures (ledger R22) name the native recipe - # models. The schema modules are not autodoc'd: documenting them raises - # ~100 nitpick warnings from pydantic internals, and the user-facing - # contract is the generated field tables (docs/fields.md); so the rendered - # type hints have no link target (re/07 P-22; re/10). + # The GSAS-II extractors' signatures name the native recipe models. The + # schema modules are not autodoc'd (documenting them raises ~100 nitpick + # warnings from pydantic internals, and the user-facing contract is the + # generated field tables), so these type hints have no link target. ('py:class', 'powderline.gateways.gsasii.schema.GsasiiRietveldRecipe'), ('py:class', 'powderline.gateways.gsasii.schema.GsasiiSpfRecipe'), # Module docstrings of the documented public modules point at modules - # api.rst does not render as pages (schema_core, the gateway-internal - # schema / native_run modules) and at pydantic's exception, which the - # pydantic inventory indexes under pydantic_core. + # the API pages do not render (schema_core, the gateway-internal schema / + # native_run modules) and at pydantic's exception, which the pydantic + # inventory indexes under pydantic_core. ('py:mod', 'powderline.schema_core'), ('py:data', 'SUPPORTED_CORE_SCHEMAS'), ('py:mod', 'schema'), @@ -93,32 +95,45 @@ ('py:class', 'powderline.exceptions.StructuredWarning'), ] +# -- MyST (Markdown) --------------------------------------------------------- -# MyST parser settings for Markdown support myst_enable_extensions = [ - "deflist", # Definition lists - "colon_fence", # ::: fences - "dollarmath", # $$...$$ math blocks (e.g. Rwp formula in DEVELOPMENT.md) + "deflist", # definition lists + "colon_fence", # ::: fences (admonitions, cards, tabs) + "dollarmath", # $...$ and $$...$$ math ] -# Auto-generate GitHub-style heading anchors (up to H4) so in-page TOC links -# like `[Root Causes Summary](#1-root-causes-summary)` in cross-platform-guide.md -# resolve instead of raising 'myst.xref_missing'. +# GitHub-style heading anchors (up to H4), so in-page links like +# `[Bounds](#bounds)` resolve. myst_heading_anchors = 4 +# A ```mermaid fence is a diagram, the same syntax GitHub renders, so a page +# reads the same on GitHub and on the site. +myst_fence_as_directive = ["mermaid"] + source_suffix = { - '.rst': 'restructuredtext', '.md': 'markdown', } -# -- Options for HTML output ------------------------------------------------- -# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output +# -- HTML output ------------------------------------------------------------- -html_theme = 'sphinx_rtd_theme' # Read the Docs *theme*; docs are hosted on GitHub Pages +html_theme = 'pydata_sphinx_theme' html_static_path = ['_static'] +html_title = f'PowderLine {release}' # Canonical URL for the GitHub Pages site (used for sitemap/canonical links). html_baseurl = 'https://nsls2.github.io/PowderLine/' -# -- Options for autodoc ----------------------------------------------------- +html_theme_options = { + "logo": {"text": "PowderLine"}, + "github_url": "https://github.com/NSLS2/PowderLine", + "navbar_align": "left", + "header_links_before_dropdown": 8, + "show_toc_level": 2, + "navigation_with_keys": False, + "footer_start": ["copyright"], + "footer_end": ["sphinx-version", "theme-version"], +} + +# -- autodoc ----------------------------------------------------------------- autodoc_default_options = { 'members': True, @@ -126,11 +141,10 @@ 'special-members': '__init__', 'undoc-members': True, # model_config is pydantic's internal ConfigDict boilerplate (identical on - # every model, not part of the recipe schema) -- excluding it avoids ~24 - # unresolvable `py:class reference target not found: ConfigDict` warnings - # (pydantic doesn't publish ConfigDict in its intersphinx inventory). + # every model, not part of the recipe schema); excluding it avoids + # unresolvable `ConfigDict` references (pydantic does not publish + # ConfigDict in its intersphinx inventory). 'exclude-members': '__weakref__,model_config', } -# Type hints configuration autodoc_typehints = 'signature' diff --git a/docs/requirements.txt b/docs/requirements.txt index 5e7a325..d4db8db 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,9 @@ # Documentation-build-only dependencies (not in pyproject.toml). -# Versions track pixi.toml. Used by the GitHub Pages docs workflow -# (.github/workflows/docs.yml, via `pixi run docs`). +# Versions track the `dev` feature in pixi.toml. Used by the GitHub Pages docs +# workflow (.github/workflows/docs.yml, via `pixi run docs`). sphinx>=8.0,<9 -sphinx_rtd_theme>=3.0,<4 myst-parser>=4.0,<5 +pydata-sphinx-theme>=0.24.0,<0.25 +sphinx-design>=0.7.0,<0.8 +sphinxcontrib-mermaid>=2.1.1,<3 +sphinx-copybutton>=0.5.2,<0.6 diff --git a/pixi.lock b/pixi.lock index 40f06b7..fd91246 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1401,6 +1401,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/zlib-ng-2.3.3-hce19668_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.5-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-51.0-unix_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/alabaster-1.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/annotated-doc-0.0.5-pyhcf101f3_0.conda @@ -1408,6 +1409,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.15.1-pyh5ded981_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/babel-2.18.0-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/backoff-2.2.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.15.0-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.10.23-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-2.0.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-2.0.1-pyhcf101f3_0.conda @@ -1479,6 +1481,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pydantic-2.14.1-pyh5ded981_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pydantic-extra-types-2.11.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pydantic-settings-2.15.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pydata-sphinx-theme-0.24.0-pyh5ded981_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pygments-2.21.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyopengl-3.1.10-pyha804496_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.3.3-pyh5ded981_0.conda @@ -1505,13 +1508,15 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/smmap-5.0.3-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-3.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-4.0.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinx-8.2.3-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/sphinx_rtd_theme-3.1.0-pyhcf101f3_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sphinx-copybutton-0.5.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sphinx-design-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-devhelp-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.1.0-pyhd8ed1ab_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jquery-4.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jsmath-1.0.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-mermaid-2.1.1-pyh5ded981_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-2.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/starlette-1.8.0-pyh5ded981_0.conda @@ -1573,12 +1578,14 @@ environments: - pypi: https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl osx-arm64: - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.5-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/alabaster-1.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/annotated-doc-0.0.5-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/annotated-types-0.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.15.1-pyh5ded981_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/babel-2.18.0-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/backoff-2.2.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.15.0-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.10.23-hbd8a1cb_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-2.0.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-2.0.1-pyhcf101f3_0.conda @@ -1651,6 +1658,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pydantic-2.14.1-pyh5ded981_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pydantic-extra-types-2.11.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pydantic-settings-2.15.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pydata-sphinx-theme-0.24.0-pyh5ded981_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pygments-2.21.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyopengl-3.1.10-pyh534df25_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.3.3-pyh5ded981_0.conda @@ -1677,13 +1685,15 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/smmap-5.0.3-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-3.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-4.0.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinx-8.2.3-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/sphinx_rtd_theme-3.1.0-pyhcf101f3_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sphinx-copybutton-0.5.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sphinx-design-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-devhelp-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.1.0-pyhd8ed1ab_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jquery-4.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jsmath-1.0.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-mermaid-2.1.1-pyh5ded981_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-2.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/starlette-1.8.0-pyh5ded981_0.conda @@ -1857,12 +1867,14 @@ environments: - pypi: https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl win-64: - conda: https://conda.anaconda.org/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_3.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.5-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/alabaster-1.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/annotated-doc-0.0.5-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/annotated-types-0.8.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/anyio-4.15.1-pyh5ded981_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/babel-2.18.0-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/backoff-2.2.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.15.0-pyha770c72_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.10.23-h4c7d964_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cached-property-2.0.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/cached_property-2.0.1-pyhcf101f3_0.conda @@ -1934,6 +1946,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/pydantic-2.14.1-pyh5ded981_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pydantic-extra-types-2.11.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pydantic-settings-2.15.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pydata-sphinx-theme-0.24.0-pyh5ded981_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pygments-2.21.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyopengl-3.1.10-pyh7428d3b_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.3.3-pyh5ded981_0.conda @@ -1960,13 +1973,15 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/smmap-5.0.3-pyhcf101f3_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sniffio-1.3.1-pyhd8ed1ab_2.conda - conda: https://conda.anaconda.org/conda-forge/noarch/snowballstemmer-3.1.1-pyhd8ed1ab_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-4.0.1-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinx-8.2.3-pyhd8ed1ab_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/sphinx_rtd_theme-3.1.0-pyhcf101f3_2.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sphinx-copybutton-0.5.2-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sphinx-design-0.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-devhelp-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-htmlhelp-2.1.0-pyhd8ed1ab_1.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jquery-4.1-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jsmath-1.0.1-pyhd8ed1ab_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-mermaid-2.1.1-pyh5ded981_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-2.0.0-pyhd8ed1ab_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-serializinghtml-2.0.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/starlette-1.8.0-pyh5ded981_0.conda @@ -7718,6 +7733,19 @@ packages: run_exports: {} size: 8144 timestamp: 1784221492234 +- conda: https://conda.anaconda.org/conda-forge/noarch/accessible-pygments-0.0.5-pyhd8ed1ab_1.conda + sha256: 1307719f0d8ee694fc923579a39c0621c23fdaa14ccdf9278a5aac5665ac58e9 + md5: 74ac5069774cdbc53910ec4d631a3999 + depends: + - pygments + - python >=3.9 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/accessible-pygments?source=hash-mapping + run_exports: {} + size: 1326096 + timestamp: 1734956217254 - conda: https://conda.anaconda.org/conda-forge/noarch/adwaita-icon-theme-51.0-unix_0.conda sha256: def01b148aa54d4de29b7d4bf58d941c8699582201db14cf78c6c89fa1666434 md5: 6f62266fb2bfc35a5071cfa1d6ffdcc9 @@ -7845,6 +7873,20 @@ packages: run_exports: {} size: 7550 timestamp: 1791654819263 +- conda: https://conda.anaconda.org/conda-forge/noarch/beautifulsoup4-4.15.0-pyha770c72_0.conda + sha256: aed4b9dcf68ec2a75e5645fed14d77fd884d38d2e52bfa6ef4b278d90cd88781 + md5: 3b261da3fe9b4168738712832410b022 + depends: + - python >=3.10 + - soupsieve >=1.2 + - typing-extensions + license: MIT + license_family: MIT + purls: + - pkg:pypi/beautifulsoup4?source=hash-mapping + run_exports: {} + size: 92704 + timestamp: 1780853175566 - conda: https://conda.anaconda.org/conda-forge/noarch/ca-certificates-2026.10.23-h4c7d964_0.conda sha256: f1d280d2618de987204102090e83733089c092dc2dd45686fd3dcbe809034f6a md5: ea9dc39dcd2815202749c1e5bc2a8973 @@ -9188,6 +9230,27 @@ packages: run_exports: {} size: 60984 timestamp: 1786146776076 +- conda: https://conda.anaconda.org/conda-forge/noarch/pydata-sphinx-theme-0.24.0-pyh5ded981_0.conda + sha256: 7a844cb4f76ebcccf9fe4f73c7bee96c9f7ab73e24fc36f9f498fd2f62061b83 + md5: 7bfb2c69283bd35b00a7b867bd6a9332 + depends: + - accessible-pygments + - babel + - beautifulsoup4 + - docutils !=0.17.0 + - jinja2 + - pygments >=2.7 + - python >=3.11 + - requests + - sphinx >=8.2,<10.0 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pydata-sphinx-theme?source=hash-mapping + run_exports: {} + size: 847915 + timestamp: 1793637713776 - conda: https://conda.anaconda.org/conda-forge/noarch/pygments-2.21.0-pyhcf101f3_0.conda sha256: f5f015ff1bc3e1b7fc08eee096b1865234189ae0b82bebdb86a5369116e42fa0 md5: 2882dee445dfa45b0c5afce3ffb7730a @@ -9726,6 +9789,17 @@ packages: run_exports: {} size: 74456 timestamp: 1780468201547 +- conda: https://conda.anaconda.org/conda-forge/noarch/soupsieve-4.0.1-pyhd8ed1ab_0.conda + sha256: 53f8475012453584cafd2c31462e915a98565bd39b866a3b61b0ef6b67eab740 + md5: 412131ebf3dab087701ff0ca7b408e02 + depends: + - python >=3.11 + license: MIT + purls: + - pkg:pypi/soupsieve?source=compressed-mapping + run_exports: {} + size: 50561 + timestamp: 1793849723725 - conda: https://conda.anaconda.org/conda-forge/noarch/sphinx-8.2.3-pyhd8ed1ab_0.conda sha256: 995f58c662db0197d681fa345522fd9e7ac5f05330d3dff095ab2f102e260ab0 md5: f7af826063ed569bb13f7207d6f949b0 @@ -9755,22 +9829,32 @@ packages: run_exports: {} size: 1424416 timestamp: 1740956642838 -- conda: https://conda.anaconda.org/conda-forge/noarch/sphinx_rtd_theme-3.1.0-pyhcf101f3_2.conda - sha256: 3581a8335a0da4de41f4344c4f1113a11fcbbb7933577ace813602e263842b02 - md5: 787507b1d0f5c00c186b2f66adacc3ab +- conda: https://conda.anaconda.org/conda-forge/noarch/sphinx-copybutton-0.5.2-pyhd8ed1ab_1.conda + sha256: 8cd892e49cb4d00501bc4439fb0c73ca44905f01a65b2b7fa05ba0e8f3924f19 + md5: bf22cb9c439572760316ce0748af3713 depends: - - python >=3.10 - - sphinx >=6,<10 - - docutils >0.18,<0.23 - - sphinxcontrib-jquery >=4,<5 - - python + - python >=3.9 + - sphinx >=1.8 + license: MIT + license_family: MIT + purls: + - pkg:pypi/sphinx-copybutton?source=hash-mapping + run_exports: {} + size: 17893 + timestamp: 1734573117732 +- conda: https://conda.anaconda.org/conda-forge/noarch/sphinx-design-0.7.0-pyhd8ed1ab_0.conda + sha256: 7f8437a97e6311bebf230cfd2ae3c5bdb2230e681c41daebdb894280bf8b4ab6 + md5: 28eddfb8b9ecdd044a6f609f985398a7 + depends: + - python >=3.11 + - sphinx >=7,<10 license: MIT license_family: MIT purls: - - pkg:pypi/sphinx-rtd-theme?source=hash-mapping + - pkg:pypi/sphinx-design?source=hash-mapping run_exports: {} - size: 6616185 - timestamp: 1785314437148 + size: 931118 + timestamp: 1769032711360 - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-applehelp-2.0.0-pyhd8ed1ab_1.conda sha256: d7433a344a9ad32a680b881c81b0034bc61618d12c39dd6e3309abeffa9577ba md5: 16e3f039c0aa6446513e94ab18a8784b @@ -9810,18 +9894,6 @@ packages: run_exports: {} size: 32895 timestamp: 1733754385092 -- conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jquery-4.1-pyhd8ed1ab_1.conda - sha256: 69c08d18663b57ebc8e4187c64c8d29b10996bb465a515cd288d87b6f2f52a5e - md5: 403185829255321ea427333f7773dd1f - depends: - - python >=3.9 - - sphinx >=1.8 - license: 0BSD AND MIT - purls: - - pkg:pypi/sphinxcontrib-jquery?source=hash-mapping - run_exports: {} - size: 112964 - timestamp: 1734344603903 - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-jsmath-1.0.1-pyhd8ed1ab_1.conda sha256: 578bef5ec630e5b2b8810d898bbbf79b9ae66d49b7938bcc3efc364e679f2a62 md5: fa839b5ff59e192f411ccc7dae6588bb @@ -9834,6 +9906,21 @@ packages: run_exports: {} size: 10462 timestamp: 1733753857224 +- conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-mermaid-2.1.1-pyh5ded981_1.conda + sha256: ff1ab95f8221fc6a9a550c0891cdccc4239f1515b347824320cb18aec5a8dc5e + md5: e0f40dce19f248bd466e2251bf1c7803 + depends: + - python >=3.11 + - pyyaml + - sphinx + - python + license: BSD-2-Clause + license_family: BSD + purls: + - pkg:pypi/sphinxcontrib-mermaid?source=hash-mapping + run_exports: {} + size: 23782 + timestamp: 1788294362671 - conda: https://conda.anaconda.org/conda-forge/noarch/sphinxcontrib-qthelp-2.0.0-pyhd8ed1ab_1.conda sha256: c664fefae4acdb5fae973bdde25836faf451f41d04342b64a358f9a7753c92ca md5: 00534ebcc0375929b45c3039b5ba7636 diff --git a/pixi.toml b/pixi.toml index 3149bad..84a4f01 100644 --- a/pixi.toml +++ b/pixi.toml @@ -154,9 +154,12 @@ test-easydiffraction = { cmd = 'pytest -m "easydiffraction"', env = { PYTHONPATH # --------------------------------------------------------------------------- [feature.dev.dependencies] sphinx = ">=8.0.0,<9" -sphinx_rtd_theme = ">=3.0.0,<4" myst-parser = ">=4.0.0,<5" gitpython = ">=3.1.44,<4" +pydata-sphinx-theme = ">=0.24.0,<0.25" +sphinx-design = ">=0.7.0,<0.8" +sphinxcontrib-mermaid = ">=2.1.1,<3" +sphinx-copybutton = ">=0.5.2,<0.6" [feature.dev.tasks] # The full gauntlet: every marker, every engine, before a merge (`pixi run From 243cb0bcf6d3750114acc29424f4dacba7135988 Mon Sep 17 00:00:00 2001 From: acorrao <95718914+AdamCorrao@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:44:24 -0500 Subject: [PATCH 02/18] docs: add the EB/KI catalogue and the example figures, with checks docs/catalog/*.yaml holds the public facts of every engine behaviour (EB-NN) and known issue (KI-NN); scripts/gen_catalog.py renders the per-engine entries, the ID index and the known-issues page, each ID with its own anchor. scripts/gen_example_figures.py draws deterministic SVG fit plots from the committed fit profiles for the examples gallery. New pixi tasks catalog-check and figures-check run in CI beside fields-check. Assisted-by: claude-code:claude-opus-5-5 --- .github/workflows/ci.yaml | 4 + docs/_generated/eb_all.md | 53 + docs/_generated/eb_easydiffraction.md | 374 +++++ docs/_generated/eb_gsasii.md | 321 ++++ docs/_generated/eb_topas.md | 267 +++ docs/_generated/examples/example_DRX_33.svg | 29 + .../examples/example_DRX_33_anisoADP.svg | 29 + .../examples/example_DRX_33_atomrefine.svg | 29 + .../example_DRX_33_easydiffraction.svg | 29 + .../examples/example_DRX_33_simulation.svg | 29 + .../examples/example_DRX_33_strainonly.svg | 29 + docs/_generated/examples/example_LaB6.svg | 29 + .../examples/example_LaB6_easydiffraction.svg | 29 + .../examples/example_LaB6_ionic.svg | 29 + .../example_LaB6_ionic_easydiffraction.svg | 29 + .../examples/example_LaB6_simulation.svg | 29 + .../examples/example_LaB6_singlepeakfit.svg | 29 + ...aB6_temperature_series_easydiffraction.svg | 29 + .../examples/example_LaB6_textured.svg | 29 + .../example_LaB6_textured_easydiffraction.svg | 29 + docs/_generated/ids.md | 118 ++ docs/_generated/known_issues.md | 169 ++ docs/catalog/engine_behaviour.yaml | 1439 +++++++++++++++++ docs/catalog/known_issues.yaml | 342 ++++ pixi.toml | 5 + scripts/gen_catalog.py | 368 +++++ scripts/gen_example_figures.py | 269 +++ 27 files changed, 4164 insertions(+) create mode 100644 docs/_generated/eb_all.md create mode 100644 docs/_generated/eb_easydiffraction.md create mode 100644 docs/_generated/eb_gsasii.md create mode 100644 docs/_generated/eb_topas.md create mode 100644 docs/_generated/examples/example_DRX_33.svg create mode 100644 docs/_generated/examples/example_DRX_33_anisoADP.svg create mode 100644 docs/_generated/examples/example_DRX_33_atomrefine.svg create mode 100644 docs/_generated/examples/example_DRX_33_easydiffraction.svg create mode 100644 docs/_generated/examples/example_DRX_33_simulation.svg create mode 100644 docs/_generated/examples/example_DRX_33_strainonly.svg create mode 100644 docs/_generated/examples/example_LaB6.svg create mode 100644 docs/_generated/examples/example_LaB6_easydiffraction.svg create mode 100644 docs/_generated/examples/example_LaB6_ionic.svg create mode 100644 docs/_generated/examples/example_LaB6_ionic_easydiffraction.svg create mode 100644 docs/_generated/examples/example_LaB6_simulation.svg create mode 100644 docs/_generated/examples/example_LaB6_singlepeakfit.svg create mode 100644 docs/_generated/examples/example_LaB6_temperature_series_easydiffraction.svg create mode 100644 docs/_generated/examples/example_LaB6_textured.svg create mode 100644 docs/_generated/examples/example_LaB6_textured_easydiffraction.svg create mode 100644 docs/_generated/ids.md create mode 100644 docs/_generated/known_issues.md create mode 100644 docs/catalog/engine_behaviour.yaml create mode 100644 docs/catalog/known_issues.yaml create mode 100644 scripts/gen_catalog.py create mode 100644 scripts/gen_example_figures.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 253f705..9630e38 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -57,6 +57,10 @@ jobs: run: pixi run test - name: Generated field tables and support matrix are current run: pixi run fields-check + - name: EB/KI catalogue fragments are current and every cited ID has an entry + run: pixi run catalog-check + - name: Example figures are current + run: pixi run figures-check gsasii: # GSAS-II suite incl. the slow regressions against the committed references. diff --git a/docs/_generated/eb_all.md b/docs/_generated/eb_all.md new file mode 100644 index 0000000..455d3d7 --- /dev/null +++ b/docs/_generated/eb_all.md @@ -0,0 +1,53 @@ + + +### Cell, ADPs and occupancy + +(eb-50)= +#### EB-50 · A Uaniso that is not positive definite is used as given + +{bdg-warning}`warned` · GSAS-II 5.7.9; easydiffraction 0.21.1; TOPAS 6 unverified + +**What happens:** GSAS-II has no positive-definiteness check. easydiffraction computes a finite pattern for such a tensor, and for a negative ADP. TOPAS is unverified. + +**PowderLine does:** Core uses the tensor as stated and warns when it has a principal value ≤ 0. + +**You see:** `uaniso_not_positive_definite` + +**Note:** The public account cites {ref}`EB-46 ` and EB-50 together for easydiffraction's finite pattern. + +### Scattering types + +(eb-13)= +#### EB-13 · gemmi strips the charge from an element symbol + +{bdg-success}`handled` · gemmi 0.7.5 + +**What happens:** gemmi, which core uses for symmetry and elements, strips the charge from a symbol: `Fe3+` becomes Fe. + +**PowderLine does:** `element` takes a bare element symbol only. An ion in `element` is refused, never stripped. Ions are stated in `scattering_types`, in the CIF spelling. + +**You see:** validation error + +(eb-25)= +#### EB-25 · Each engine spells ions its own way + +{bdg-success}`handled` · GSAS-II 5.7.9; TOPAS 6; easydiffraction 0.21.1 (CrysPy 0.13.0) + +**What happens:** GSAS-II and TOPAS spell an ion `Fe+3`. CrysPy spells it `Fe3+`, the CIF spelling. + +**PowderLine does:** You write every ion in the CIF spelling, in `scattering_types`. Each gateway writes its engine's spelling. `Fe+3` in a recipe is an error that names `Fe3+`. + +**You see:** validation error + +**Note:** The public account cites {ref}`EB-13 ` and EB-25 together for ion notation; this entry holds the spelling difference between engines. + +### Statistics + +(eb-88)= +#### EB-88 · A symmetric profile shifts the refined cell + +{bdg-secondary}`documented` · GSAS-II 5.7.9; easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + +**What happens:** Lattice parameters agree between GSAS-II and easydiffraction when axial asymmetry is modelled physically (FCJ). With a symmetric profile they shift by +67 to +91 ppm, whichever calculator runs it. + +**PowderLine does:** Documented on the cross-engine comparison. Rwp compares models fitted to the same points, not engines. diff --git a/docs/_generated/eb_easydiffraction.md b/docs/_generated/eb_easydiffraction.md new file mode 100644 index 0000000..d5b7d88 --- /dev/null +++ b/docs/_generated/eb_easydiffraction.md @@ -0,0 +1,374 @@ + + +### Space groups + +(eb-42)= +#### EB-42 · A space group is a short name plus a setting code + +{bdg-success}`handled` · easydiffraction 0.21.1 + +**easydiffraction does:** easydiffraction takes one of 230 short names and a separate setting code: 531 settings in all. + +**PowderLine does:** The easydiffraction gateway translates the canonical name into a short name and a setting code. It refuses what easydiffraction cannot express. + +**You see:** validation error + +(eb-80)= +#### EB-80 · Each calculator misreads some settings + +{bdg-success}`handled` · easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + +**easydiffraction does:** CrysFML is handed the short symbol alone. It misreads almost every non-default setting: unique axis, cell choice, `:R` and some origin-1 groups. CrysPy computes `P 31` with the operators of `P 32`. + +**PowderLine does:** Validation accepts, per calculator, only the settings whose computation passed for a general position and for every special site: 428 of the 531 for CrysPy, 230 for CrysFML. The error names a setting of the same group that the calculator computes, or the other calculator. + +**You see:** validation error + +(eb-84)= +#### EB-84 · CrysPy misreads 41 settings and crashes on 53 + +{bdg-success}`handled` · easydiffraction 0.21.1 (CrysPy 0.13.0) + +**easydiffraction does:** CrysPy computes 41 settings as another setting, and crashes on 53. + +**PowderLine does:** Validation accepts only the settings CrysPy computes correctly (428 of 531). The error names a setting of the same group that CrysPy computes, or the crysfml calculator. + +**You see:** validation error + +**Note:** The public account cites EB-84 after both facts in one sentence; whether the 41 misread settings belong to EB-84 or {ref}`EB-80 ` is not stated separately. + +(eb-83)= +#### EB-83 · CrysPy resets the unique angle of some monoclinic cells to 90° + +{bdg-success}`handled` · easydiffraction 0.21.1 (CrysPy 0.13.0) + +**easydiffraction does:** For c-unique and a-unique monoclinic cells, CrysPy resets the unique angle to 90°. + +**PowderLine does:** These settings are not among those validation accepts for the cryspy calculator. The error names a setting of the same group that CrysPy computes, or the crysfml calculator. + +**You see:** validation error + +### Special positions and sites + +(eb-12)= +#### EB-12 · CrysPy rationalizes and snaps coordinates + +{bdg-success}`handled` · easydiffraction 0.21.1 (CrysPy 0.13.0) + +**easydiffraction does:** CrysPy rationalizes each coordinate to a denominator of at most 10⁵. It snaps 5-decimal thirds. + +**PowderLine does:** Core hands over exact values, so CrysPy's own tolerance never matters. + +(eb-47)= +#### EB-47 · Wyckoff positions are detected only when the structure is built + +{bdg-success}`handled` · easydiffraction 0.21.1 + +**easydiffraction does:** easydiffraction applies site symmetry itself. It detects each atom's Wyckoff position when the structure is built, and never re-detects during a fit. + +**PowderLine does:** The gateway hands over core's exact values. After the build, it checks easydiffraction's multiplicity against core's for every atom. A refined coordinate that ends near a special position is reported through the output recipe ({ref}`KI-18 `). + +**You see:** `output_recipe_not_valid` + +(eb-74)= +#### EB-74 · easydiffraction fixes special coordinates and re-detects them on an edit + +{bdg-success}`handled` · easydiffraction 0.21.1 + +**easydiffraction does:** easydiffraction marks fixed coordinates and ignores a `free` flag on them. It re-detects the Wyckoff position on a user edit with tolerance 1e-3, so an off-special coordinate can change the multiplicity. + +**PowderLine does:** The gateway hands over core's exact values. After the build, it checks easydiffraction's multiplicity against core's for every atom. + +**Note:** The public account cites {ref}`EB-47 ` and EB-74 together for easydiffraction's site-symmetry handling; the part that is EB-74's own is not stated separately. + +(eb-77)= +#### EB-77 · CrysPy computes some special positions wrongly + +{bdg-success}`handled` · easydiffraction 0.21.1 (CrysPy 0.13.0) + +**easydiffraction does:** CrysPy computes wrong structure factors for some equivalent writings of a special position with a free coordinate. LaB6's B written as (½, ½, x) instead of the template's (x, ½, ½) gives intensities off by up to ×2. It also computes special positions wrongly in nine origin-choice-1 settings. General positions and fixed special positions are right. + +**PowderLine does:** With `calculator: "cryspy"`, an atom on such a site must be written in the pattern of easydiffraction's Wyckoff template. The builder refuses anything else before the run and names the position to write. A position that easydiffraction only stores translated by a lattice vector is the same atom: it is accepted with a warning, and reports keep the recipe's writing. The nine settings are not accepted for the cryspy calculator. CrysFML has no such rule. + +**You see:** validation error, `easydiffraction_position_translated` + +### Cell, ADPs and occupancy + +(eb-23)= +#### EB-23 · CrysFML ignores anisotropic ADPs + +{bdg-success}`handled` · easydiffraction 0.21.1 (CrysFML 0.8.0) + +**easydiffraction does:** easydiffraction supports anisotropic ADPs, but CrysFML is handed only B = 8π²·Ueq. A `Uaniso` has no effect with CrysFML. + +**PowderLine does:** `Uaniso` with the crysfml calculator is a validation error. + +**You see:** validation error + +(eb-86)= +#### EB-86 · A refined Uij can land in the wrong atom's column + +{bdg-success}`handled` · easydiffraction 0.21.1 (CrysPy 0.13.0) + +**easydiffraction does:** With CrysPy, easydiffraction writes a refined Uij into the wrong atom's column whenever an isotropic atom precedes the anisotropic one. The refined tensor then has no effect. + +**PowderLine does:** The gateway hands the anisotropic atoms to easydiffraction first. Reports keep the recipe's order. A test proves that a refined Uij moves. + +(eb-46)= +#### EB-46 · Negative ADPs are refused at input + +{bdg-success}`handled` · easydiffraction 0.21.1 + +**easydiffraction does:** easydiffraction refuses a negative `Uiso` or U11, U22, U33 at input. Its physical limit for them is 0–10 Ų. + +**PowderLine does:** An `easydiffraction.*` recipe may not state a negative `Uiso` or U11, U22, U33: validation error. + +**You see:** validation error + +(eb-22)= +#### EB-22 · Occupancy outside [0, 1] is refused + +{bdg-success}`handled` · easydiffraction 0.21.1 + +**easydiffraction does:** easydiffraction refuses an occupancy outside [0, 1]. + +**PowderLine does:** Core requires 0 ≤ occupancy ≤ 1 for every engine, so the recipe is refused before easydiffraction sees it. + +**You see:** validation error + +### Scattering types + +(eb-95)= +#### EB-95 · An ion CrysPy has no form factor for becomes the neutral atom + +{bdg-success}`handled` · easydiffraction 0.21.1 (CrysPy 0.13.0) + +**easydiffraction does:** easydiffraction accepts any CIF type symbol, such as `O2-`. When CrysPy has no form factor for the ion, it uses the neutral atom with a log warning only. CrysPy has 111 ions: `O1-` but no `O2-`. + +**PowderLine does:** An ion outside CrysPy's table is a validation error that names the forms CrysPy has. + +**You see:** validation error + +(eb-96)= +#### EB-96 · CrysFML computes neutral atoms only + +{bdg-success}`handled` · easydiffraction 0.21.1 (CrysFML 0.8.0) + +**easydiffraction does:** easydiffraction strips every charge before handing the type to CrysFML. + +**PowderLine does:** `scattering_types` with the crysfml calculator is a validation error. Use the cryspy calculator for ions. + +**You see:** validation error + +### Names + +(eb-45)= +#### EB-45 · Atom labels are restricted and datablock names lowercased + +{bdg-success}`handled` · easydiffraction 0.21.1 + +**easydiffraction does:** easydiffraction accepts atom labels of the form `[A-Za-z_][A-Za-z0-9_]*` only. It lowercases datablock names. + +**PowderLine does:** The core name rule makes every name easydiffraction would reject or merge invalid before it runs. + +**You see:** validation error + +### Bounds and limits + +(eb-71)= +#### EB-71 · Unbounded sides get physical limits, and starts are moved silently + +{bdg-success}`handled` · easydiffraction 0.21.1 + +**easydiffraction does:** When a fit starts, easydiffraction gives an unbounded side its physical limit: cell lengths 0–30 Å, angles 0–180°, occupancy 0–1, `adp_iso` and U11/U22/U33 0–10 Ų, scale, wavelength and μR ≥ 0. It refuses a value outside a physical limit at input, even when fixed, so no cell over 30 Å can be modelled. lmfit moves a start outside its bounds onto the bound without a message. + +**PowderLine does:** A stated bound replaces the physical limit on its side, and `null` keeps it. A value outside a physical limit is a validation error. A refined value that ends within 1e-6 (relative) of a bound or physical limit gets a warning. For a limit of 0, the test is within 1e-6 of the starting value. + +**You see:** validation error, `easydiffraction_parameter_at_limit` + +### Peak profile and instrument + +(eb-75)= +#### EB-75 · The X-ray corrections easydiffraction models, and its polarization + +{bdg-success}`handled` · easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + +**easydiffraction does:** Both calculators model the Kα doublet, the Lorentz-polarization factor 1 − p + p cos²2θ_m cos²2θ, the 2θ offset, sample displacement and transparency, and Hewat cylinder absorption. A second wavelength alone does nothing. 2θ_m acts only with p > 0. GSAS-II's polarization P is easydiffraction's 1 − p: copying P to p is a 6.6 % intensity error at 15° 2θ. + +**PowderLine does:** Every correction is a recipe slot; left out means not modelled. The schema requires both Kα2 fields or neither. The hand-derived per-engine recipes state p = 1 − P. + +**You see:** validation error + +(eb-87)= +#### EB-87 · Negative peak widths give a wrong pattern without a message + +{bdg-success}`handled` · easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + +**easydiffraction does:** Where U tan²θ + V tanθ + W < 0, CrysFML drops the peaks and CrysPy distorts them. Where X tanθ + Y/cosθ < 0, both give a finite pattern 12–24 % off. + +**PowderLine does:** A stated profile that makes a width negative is a validation error. The check covers the handed points and a 0.01° grid over the window widened by 1° plus the position corrections. A refined profile that does so gets a warning. The LaB6 CrysPy pseudo-Voigt fit refines X to −0.098, negative from 11.2° on. + +**You see:** validation error, `easydiffraction_negative_width` + +**Seen in:** `example_LaB6_ionic` + +### Preferred orientation + +(eb-97)= +#### EB-97 · Preferred orientation reaches CrysPy only, under the right key + +{bdg-success}`handled` · easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + +**easydiffraction does:** Only the CrysPy calculator reads the experiment's preferred-orientation rows. CrysFML's pattern does not change with `march_r`, without a message. CrysPy also ignores a row whose `structure_id` is not the lowercase linked structure name, silently. + +**PowderLine does:** `preferred_orientation` with `calculator: "crysfml"` is a validation error. The builder keys the row by the structure name easydiffraction uses. + +**You see:** validation error + +(eb-98)= +#### EB-98 · CrysPy's March correction is Dollase's times r^1.5 + +{bdg-secondary}`documented` · easydiffraction 0.21.1 (CrysPy 0.13.0) + +**easydiffraction does:** CrysPy's texture correction is not volume-normalised, so every textured-to-untextured intensity ratio carries the constant r^1.5. With `march_random_fract` = 0 the scale absorbs it, and the refined `march_r` is the same r as GSAS-II's. With f > 0 the correction is (1 − f)·r^1.5·A + f, and the refined r and f are not GSAS-II's quantities. + +**PowderLine does:** Documented. A test compares normalised ratios with GSAS-II's March–Dollase correction. + +### Data and the fit window + +(eb-70)= +#### EB-70 · Only the handed points are calculated + +{bdg-secondary}`documented` · easydiffraction 0.21.1 + +**easydiffraction does:** easydiffraction calculates only the points it is handed, and maps its Chebyshev basis over them. Reflection tails from beyond the window are lost: at most 3e-4 σ at the refined scale. + +**PowderLine does:** The gateway hands easydiffraction exactly the weighted points inside the window, so it fits the points core scores. The lost tails are documented. + +(eb-73)= +#### EB-73 · A σ below 1e-4 becomes 1.0 silently + +{bdg-success}`handled` · easydiffraction 0.21.1 + +**easydiffraction does:** On every input path, easydiffraction replaces σ < 1e-4 with σ = 1.0. + +**PowderLine does:** Weights above 1e8 on handed points are a validation error. + +**You see:** validation error + +(eb-91)= +#### EB-91 · Negative measured intensities are refused + +{bdg-success}`handled` · easydiffraction 0.21.1 + +**easydiffraction does:** easydiffraction refuses negative measured intensities. + +**PowderLine does:** A negative intensity on a handed point is a validation error. + +**You see:** validation error + +(eb-72)= +#### EB-72 · The ASCII data reader rounds 2θ to 4 decimals + +{bdg-success}`handled` · easydiffraction 0.21.1 + +**easydiffraction does:** easydiffraction's ASCII data reader rounds 2θ to 4 decimals. + +**PowderLine does:** The gateway hands the data as a CIF loop at full precision. + +(eb-89)= +#### EB-89 · The CIF loop cannot read exponent notation + +{bdg-success}`handled` · easydiffraction 0.21.1 + +**easydiffraction does:** easydiffraction's CIF loop reader cannot read numbers in exponent notation. + +**PowderLine does:** The gateway writes every number in the data loop in positional notation. + +### Statistics + +(eb-81)= +#### EB-81 · A parameter without effect gets no ESD + +{bdg-warning}`warned` · easydiffraction 0.21.1 + +**easydiffraction does:** easydiffraction's free-parameter count is lmfit's. A parameter without effect gets no ESD. Its own `prof_wr_factor` is 1/σ²-weighted. + +**PowderLine does:** `parameters_varied` is lmfit's count. The engine's own statistics stay under `engine_details`. + +**You see:** `easydiffraction_parameters_not_varied` + +### Calculators + +(eb-76)= +#### EB-76 · The two calculators compute different peak types + +{bdg-success}`handled` · easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + +**easydiffraction does:** CrysPy computes `cwl-pseudo-voigt` and the Bérar–Baldinozzi empirical asymmetry. CrysFML computes `cwl-pseudo-voigt` and the Thompson–Cox–Hastings profile with FCJ axial asymmetry. Both pseudo-Voigts are the TCH pseudo-Voigt with Caglioti Gaussian and X tanθ + Y/cosθ Lorentzian widths. `cutoff_fwhm` acts in CrysPy only; CrysFML uses a fixed 30-FWHM window. + +**PowderLine does:** The peak type must be one the chosen calculator computes. Otherwise validation fails. + +**You see:** validation error + +(eb-79)= +#### EB-79 · CrysFML reports no reflection list + +{bdg-warning}`warned` · easydiffraction 0.21.1 (CrysFML 0.8.0) + +**easydiffraction does:** CrysFML reports no reflection list. + +**PowderLine does:** No peak list reports are written with CrysFML, and the result says so. + +**You see:** `easydiffraction_reflections_not_available` + +(eb-90)= +#### EB-90 · CrysFML refinements depend on their starting values + +{bdg-warning}`warned` · easydiffraction 0.21.1 (CrysFML 0.8.0) + +**easydiffraction does:** CrysFML computes in single precision, below lmfit's finite-difference step. A CrysFML refinement stops at a point that depends on its starting values. Starts changed by 1e-6 (relative) gave Rwp 6.79 % against 6.27 %, with cells stable to about 1e-5. + +**PowderLine does:** Every CrysFML refinement carries a warning. The calculator stays available because you choose it, and PowderLine states the difference. + +**You see:** `easydiffraction_crysfml_start_dependent` + +(eb-78)= +#### EB-78 · CrysFML's integrated intensities drift with angle + +{bdg-secondary}`documented` · easydiffraction 0.21.1 (CrysFML 0.8.0) + +**easydiffraction does:** Relative to m·|F|²·Lorentz, CrysFML's integrated intensities drift by −4 % from 3° to 15° 2θ. This biases `Uiso` by about +0.001 Ų. + +**PowderLine does:** The calculator is recorded in every result. + +(eb-93)= +#### EB-93 · On the temperature series, easydiffraction refines La's Uiso low + +{bdg-secondary}`documented` · easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + +**easydiffraction does:** On GSAS-II-calculated LaB6 patterns, easydiffraction refines La's Uiso 10–22 % low. B's Uiso, the thermal expansion and GoF ≈ 1 agree. The cause is the series' profile: GSAS-II clamps its Lorentzian width to 0.001 centideg above about 10.7°, so the high-angle peaks are pure Gaussian. Neither a symmetric pseudo-Voigt nor TCH can represent that, and `Uiso` absorbs the mismatch. Simulated with a profile easydiffraction can represent, CrysPy's La Uiso is 0.00863 against 0.00858 programmed. CrysFML's is 0.00823 (−4 %), the size of its intensity drift ({ref}`EB-78 `). + +**PowderLine does:** Documented, no code change. Across engines, compare cells, with the {ref}`EB-88 `-size offsets in mind, and the expansion. Absolute `Uiso` agrees only when the profile models agree. + +**Seen in:** `example_LaB6_temperature_series` + +### Platform and installation + +(eb-48)= +#### EB-48 · An environment variable can turn errors into log messages + +{bdg-success}`handled` · easydiffraction 0.21.1 + +**easydiffraction does:** With `ED_LOG_REACTION=WARN` in the environment, easydiffraction downgrades its errors to log messages. + +**PowderLine does:** The gateway sets the reaction to `RAISE` itself, whatever the environment. + +(eb-82)= +#### EB-82 · Importing easydiffraction queries PyPI + +{bdg-secondary}`documented` · easydiffraction 0.21.1 + +**easydiffraction does:** Importing easydiffraction imports `diffpy.pdffit2`, whose version module queries PyPI. Offline, it prints a warning and continues. + +**PowderLine does:** Nothing. The import, and the run, continue. diff --git a/docs/_generated/eb_gsasii.md b/docs/_generated/eb_gsasii.md new file mode 100644 index 0000000..48730c3 --- /dev/null +++ b/docs/_generated/eb_gsasii.md @@ -0,0 +1,321 @@ + + +### Space groups + +(eb-03)= +#### EB-03 · Origin choice 1 is silently ignored + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II's own space-group lookup tolerates `:H` and `:2` and silently ignores `:1`. A recipe meant in origin choice 1 runs in origin choice 2, on a different structure. + +**PowderLine does:** The gsasii gateway refuses every origin-choice-1 setting (33 settings). To use one, transform the structure to a standard setting. + +**You see:** validation error + +(eb-04)= +#### EB-04 · A rhombohedral symbol means hexagonal axes unless it ends in R + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II reads `R -3 m` and `R -3 m:H` on hexagonal axes. `R -3 m R` or `:R` gives rhombohedral axes. + +**PowderLine does:** Core requires the canonical name, which states `:H` or `:R` explicitly. A bare rhombohedral symbol is an error that lists both choices. The gsasii gateway writes a trailing ` R` for `:R` and drops `:H`. + +**You see:** validation error + +(eb-41)= +#### EB-41 · Some spellings are not found, and two settings are read as another + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II reads spaced symbols. Its lookup returns nothing for spellings such as `R-3m:H` or `Fd-3m:2`. It reads `P 21 n m` and `A b a m` as another setting, whose general position has a different orbit. + +**PowderLine does:** The gsasii gateway translates the canonical name: a spaced symbol, ` R` for `:R`, `:H` and `:2` dropped. It refuses 9 settings beyond origin choice 1: 7 that GSAS-II does not parse and 2 that it misreads. A general-position orbit test over every accepted setting proves the translation. + +**You see:** validation error + +### Special positions and sites + +(eb-01)= +#### EB-01 · GSAS-II uses its own special-position tolerance + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II compares symmetry images within 2e-5 + 1e-5·|x|, and not modulo 1. + +**PowderLine does:** Core decides special positions, not the engine. An atom within 5e-6 (fractional) of a special position is on it. An atom 5e-6 to 2e-3 off is a validation error. Derived values are held exactly in the validated model, so GSAS-II's tolerance never matters. + +**You see:** `symmetry_value_derived`, validation error + +(eb-02)= +#### EB-02 · A coordinate next to a cell edge gets a wrong multiplicity + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** A coordinate just below an integer gets multiplicity 0. A coordinate just above 0 gets an invalid multiplicity. + +**PowderLine does:** This cannot be reached: such coordinates are exact constants in the validated model. + +(eb-40)= +#### EB-40 · Some 2-fold sites in R32, R-3m and R-3c are misread + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** On hexagonal axes, GSAS-II reads some orientations of these 2-fold sites, such as (0, 1/3, 1/6), as a general position with doubled multiplicity. With nothing refined, it computes four times the Bragg intensity of the equivalent (1/3, 0, 1/6). + +**PowderLine does:** After phase setup, the gsasii gateway compares GSAS-II's site symmetry and multiplicity with core's for every atom. A wrong multiplicity always stops the run, even in a simulation. The error names an equivalent position GSAS-II reads correctly. PowderLine never substitutes the position silently, because refined coordinates would then differ from the recipe. + +**You see:** `unusable_site` + +(eb-05)= +#### EB-05 · Some site symmetries cannot be named, and refining them crashes + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II cannot name the symmetry of some site orientations (`'sp'`), for example F432 at (0, y, 0). It crashes when a coordinate or `Uaniso` of such an atom is refined. + +**PowderLine does:** After phase setup, the gsasii gateway stops with an error naming an equivalent position GSAS-II can name. It does so only when a coordinate or `Uaniso` of that atom is refined. + +**You see:** `unusable_site` + +### Cell, ADPs and occupancy + +(eb-06)= +#### EB-06 · A fixed cell is not checked against its symmetry + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II enforces cell symmetry only while the cell is refined. A fixed, inconsistent cell is used as given and gives wrong peak positions. + +**PowderLine does:** Core refuses a cell that does not fit the space group, within 1e-9, in every engine. The error gives the value to write ({ref}`KI-01 `). + +**You see:** validation error + +(eb-07)= +#### EB-07 · Uij site-symmetry constraints apply only while refining + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II applies the site-symmetry constraints on Uij when refining. It stores the values as given. + +**PowderLine does:** Core requires `Uij` values that respect the site symmetry, before any engine runs. + +**You see:** validation error + +(eb-51)= +#### EB-51 · Negative ADPs are carried and refined + +{bdg-secondary}`documented` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II carries negative `Uiso` and `Uij` values and refines from them. + +**PowderLine does:** Core has no sign rule for ADPs: a negative ADP can compensate for an unmodelled effect. + +(eb-20)= +#### EB-20 · Any occupancy is accepted + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II accepts any occupancy value. + +**PowderLine does:** Core requires 0 ≤ occupancy ≤ 1 for every engine. + +**You see:** validation error + +### Scattering types + +(eb-94)= +#### EB-94 · An ion in CIF spelling is read as hydrogen + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II reads the CIF spelling `La3+` as hydrogen (Z = 1), silently. Its own spelling is `La+3`. Its form-factor table has 114 ions. + +**PowderLine does:** You write `La3+` in `scattering_types`. The gsasii gateway writes GSAS-II's spelling into the atom record. It refuses an ion GSAS-II has no form factor for. + +**You see:** validation error + +**Seen in:** `example_LaB6_ionic` + +### Names + +(eb-44)= +#### EB-44 · GSAS-II rewrites phase names + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II strips surrounding whitespace from a phase name and replaces non-ASCII characters with `_`. The settings of a renamed phase are then skipped silently. A `/` in the name breaks the report file write. + +**PowderLine does:** The core name rule makes every such name invalid before an engine runs: `^[A-Za-z][A-Za-z0-9_]*$`, at most 64 characters, no Python keyword, distinct ignoring case. + +**You see:** validation error + +### Bounds and limits + +(eb-30)= +#### EB-30 · Parameter limits apply only after all cycles + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II applies parameter limits only after all cycles of a `Refine` call. An out-of-range value is set to the limit with ESD 0 and frozen for the next call. The reported values, ESDs and Rwp belong to the unclamped fit. + +**PowderLine does:** `gsasii.*` recipes expose no bounds. A 4-element parameter list is a validation error ({ref}`KI-06 `). + +**You see:** validation error + +(eb-31)= +#### EB-31 · Cell limits act on the reciprocal metric + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** Cell limits must be set on the reciprocal-metric terms. They map exactly onto direct-cell boxes for only four of the seven crystal systems. + +**PowderLine does:** `gsasii.*` recipes expose no bounds. A 4-element parameter list is a validation error. + +**You see:** validation error + +### Defaults + +(eb-36)= +#### EB-36 · A new project adds broadening and background you did not ask for + +{bdg-warning}`warned` · GSAS-II 5.7.9 + +**GSAS-II does:** A new GSAS-II project gets isotropic size 1 µm, microstrain 1000 and a constant 1.0 Chebyshev background, all fixed. + +**PowderLine does:** Size and strain left out mean 10 µm and 0 microstrain, isotropic, fixed: no sample broadening. A background left out means none: one fixed Chebyshev term 0.0. Each default comes with a warning naming the phase and the values. State the blocks to silence the warning. + +**You see:** `gsasii_broadening_default_applied`, `gsasii_background_default_applied` + +### Peak profile and instrument + +(eb-37)= +#### EB-37 · SH/L and Peak List widths are floored silently + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II floors SH/L at 0.002 in the Rietveld and Peak List profile calculations. It floors it at 0.0005 in `GetFobsSq`, and background peaks use it as stored. Peak List peaks floor σ² and γ at 0.001. + +**PowderLine does:** `axial_divergence` below 0.002 is a validation error. So is a Peak List width below its floor. The converted LaB6 examples state 0.002 where they said 0.0005. + +**You see:** validation error + +(eb-35)= +#### EB-35 · Background peaks are evaluated with silent floors + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II evaluates background peaks with floors: intensity 0.1, σ² 0.01 centideg², γ 0.1 centideg. + +**PowderLine does:** A stated value below a floor is a validation error. A refined value below a floor makes the output recipe invalid ({ref}`KI-15 `). + +**You see:** validation error, `output_recipe_not_valid` + +**Seen in:** `example_LaB6_singlepeakfit` + +(eb-34)= +#### EB-34 · The background-peak width is a variance + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II's `BkPksig` is σ² in centideg², although its GUI calls it `sig` and its prose calls it "Gaussian width". + +**PowderLine does:** The recipe field is named `pv_gaussian_sigma_sq`. + +(eb-39)= +#### EB-39 · Exported peak widths use other units, and some formulas are wrong + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II's peak `int` is an integrated area. Its exporters write σ in degrees while the project holds σ² in centideg², and several exporter formulas are wrong. + +**PowderLine does:** PowderLine reads the project directly and uses none of those exporters. + +### Data and the fit window + +(eb-54)= +#### EB-54 · GSAS-II fits one point past the upper limit + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** When the upper limit falls between data points, GSAS-II fits and scores one point past it. With no limit, it starts at the first data point, weighted or not. Its `Nobs` counts the extra point. + +**PowderLine does:** The gateway hands GSAS-II the fit limits moved onto the first and last weighted data points inside the window. GSAS-II then fits exactly the points core scores, and its own Rwp equals core's. + +### Refinement, convergence and failures + +(eb-32)= +#### EB-32 · Refinement messages are only printed and stored + +{bdg-warning}`warned` · GSAS-II 5.7.9 + +**GSAS-II does:** `Refine` returns nothing on success. Its messages about frozen limits, parameters dropped as singular and SVD are only printed and stored in the project. + +**PowderLine does:** The gateway reads the stored messages into warnings. + +**You see:** `gsasii_refinement_message` + +(eb-33)= +#### EB-33 · The parameter count includes parameters dropped as singular + +{bdg-warning}`warned` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II's `Nvars` includes parameters it dropped as singular. + +**PowderLine does:** PowderLine reports the drop as a warning and does not subtract it. In a single-peak fit, the count is taken after the drops. + +(eb-53)= +#### EB-53 · Single-peak fitting ignores some refine flags silently + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** With the instrument profile in use, a peak's own σ² and γ flags are ignored. With per-peak widths, the U … Z flags are ignored. Refining any σ² clears the U, V, W flags. Singular parameters are dropped without a message. + +**PowderLine does:** A refine flag on the width source not in use is a validation error. Any remaining silent drop gets a warning. + +**You see:** validation error, `gsasii_parameters_not_varied` + +(eb-92)= +#### EB-92 · A peak fit fills no covariance block + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** `DoPeakFit` keeps its varied names and ESDs in the Peak List's own record, and writes the refined values in place. A peak fit therefore has no refined-parameter rows. + +**PowderLine does:** The result's `peak_fit` block reads the values and ESDs from there. The output recipe is built from it. + +(eb-38)= +#### EB-38 · A peak outside the fit window makes the peak fit fail with a bare TypeError + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** A Peak List or background peak outside the fit window makes GSAS-II's peak fit fail with a bare `TypeError`. + +**PowderLine does:** A peak outside the fit window is a validation error. A peak fit that still fails raises `EngineExecutionError` with a clear reason. + +**You see:** validation error, `peak_fit_failed` + +(eb-99)= +#### EB-99 · An aborted refinement keeps a worthless pattern + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** When no damped step lowers χ², GSAS-II aborts the refinement. It keeps the last accepted values but stores the rejected step's pattern and a wR at its 100 % cap. No statistic belongs to the parameters. GSAS-II's GUI shows this only as "Minimizer halted because chi**2 increased". A March–Dollase ratio started at exactly 1.0 in a cubic phase triggers it. + +**PowderLine does:** PowderLine is deliberately stricter: an aborted Rietveld refinement is a failure, reason `diverged`. Simulations are exempt. + +**You see:** `diverged` + +### Statistics + +(eb-52)= +#### EB-52 · GOF has two meanings, and R values are capped at 100 % + +{bdg-success}`handled` · GSAS-II 5.7.9 + +**GSAS-II does:** GSAS-II's `GOF` is √(reduced χ²) in a Rietveld refinement and the reduced χ² itself in a Peak List fit. GSAS-II caps R and wR at 100 %, so a simulation against placeholder data and a diverged refinement both read 100 %. A non-finite pattern prints wR `nan` and reports 100 %. + +**PowderLine does:** The result's `rwp`, `r_exp`, `gof` and `chi2_red` are PowderLine's, with one formula for every engine. GSAS-II's own values stay under `engine_details`. A non-finite calculated pattern is a failure, reason `diverged`. A poor fit at the cap is still reported, with core's uncapped Rwp beside it. + +**You see:** `diverged` diff --git a/docs/_generated/eb_topas.md b/docs/_generated/eb_topas.md new file mode 100644 index 0000000..dde3c78 --- /dev/null +++ b/docs/_generated/eb_topas.md @@ -0,0 +1,267 @@ + + +### Space groups + +(eb-11)= +#### EB-11 · A bare two-origin symbol is origin choice 1 + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** TOPAS accepts the symbols of its own table: case-insensitive, spaces allowed, or the number. `:2` selects origin choice 2. A bare two-origin symbol is origin choice 1, the opposite of GSAS-II's default. + +**PowderLine does:** The topas gateway writes TOPAS's explicit form: `:2` kept, `:1` dropped. + +(eb-43)= +#### EB-43 · A bare rhombohedral symbol means hexagonal axes + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** TOPAS reads a bare rhombohedral symbol on hexagonal axes. A trailing `r` gives rhombohedral axes. + +**PowderLine does:** The topas gateway drops `:H` and writes `:R` as a trailing `r`. + +(eb-68)= +#### EB-68 · Some settings have no TOPAS symbol, and three are misread + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** Twenty of gemmi's settings have no TOPAS symbol. TOPAS 6 reads `C 1 21 1`, `I 1 21 1` and `C 4 2 21` as another setting, dropping an origin shift. + +**PowderLine does:** The topas gateway refuses these 23 settings. The other 541 were checked operator by operator against TOPAS 6's own generated operator files. + +**You see:** validation error + +### Special positions and sites + +(eb-08)= +#### EB-08 · TOPAS compares positions within 1e-15 + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** TOPAS compares equivalent positions within 1e-15. It treats any decimal third as a general position. + +**PowderLine does:** Core holds derived values exactly. The topas writer emits fixed special coordinates as exact equations, such as `= 1/3;`. + +### Cell, ADPs and occupancy + +(eb-09)= +#### EB-09 · Cell values are used as given + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** TOPAS uses the individual `a b c al be ga` values as given. + +**PowderLine does:** Core refuses a cell that does not fit the space group, within 1e-9, in every engine. + +**You see:** validation error + +(eb-10)= +#### EB-10 · The adps keyword applies the Uij constraints + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** TOPAS's `adps` keyword applies the site-symmetry constraints on Uij. + +**PowderLine does:** Core requires `Uij` values that respect the site symmetry, before any engine runs. + +**You see:** validation error + +**Note:** Unverified. Recorded from documentation, not from a run. + +(eb-24)= +#### EB-24 · TOPAS's isotropic ADP is beq, not Uiso + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** TOPAS states the isotropic ADP as `beq` = 8π²·Uiso. + +**PowderLine does:** The recipe states `Uiso` in Ų, as in every schema. The topas gateway converts the value and its bounds to `beq` and warns once per atom. Results report `Uiso`. + +**You see:** `topas_adp_converted` + +(eb-21)= +#### EB-21 · Occupancy is limited per iteration + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** TOPAS limits `occ` to 0 … 2·value + 1 per iteration. + +**PowderLine does:** Core requires 0 ≤ occupancy ≤ 1 for every engine. + +**You see:** validation error + +### Names + +(eb-49)= +#### EB-49 · Two parameters with the same name are one parameter + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** TOPAS treats two `prm` of the same name as one parameter, so composed names could collide silently. + +**PowderLine does:** The topas writer uses index-based parameter names. The core name rule also applies. + +**Note:** Unverified beyond TOPAS's documented model. + +### Bounds and limits + +(eb-58)= +#### EB-58 · Keywords and macros carry default limits + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** TOPAS keywords carry default limits: `scale` ≥ 1e-11, `beq` in [−10, 20], cell lengths ≥ 1.5 Å, lattice ±0.5 % per iteration, among others. The TOPAS.INC macros add their own: TCHZ U–Z in [−1, 2], X and Y in [1e-4, 2], CS in [0.3, 10000] nm, Strain in [1e-4, 5], and the March–Dollase `PO` ratio min 1e-4 and max 2r + 0.5. A user `min` or `max` overrides the default on that side. A start outside a limit is moved onto it silently. + +**PowderLine does:** A stated bound replaces TOPAS's limit on that side. A `null` side keeps TOPAS's default limit. A refined start outside a fixed default limit on a `null` side is a validation error: state a bound, or start inside. A refined value within 1e-4 (relative) of a stated bound or a default limit gets a warning. At a limit of 0 the tolerance is 1e-12 absolute. + +**You see:** validation error, `topas_parameter_at_limit` + +(eb-62)= +#### EB-62 · The limit markers TOPAS writes are unreliable + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** The `_LIMIT_MIN` and `_LIMIT_MAX` markers TOPAS 6 writes are mislabelled or spurious. + +**PowderLine does:** The at-limit warning is judged from the final values and the limits the writer knows, not from TOPAS's markers. + +**You see:** `topas_parameter_at_limit` + +### Peak profile and instrument + +(eb-55)= +#### EB-55 · TCHZ takes the absolute value of its Gaussian term + +{bdg-secondary}`documented` · TOPAS 6 + +**TOPAS does:** TCHZ computes ΓG = √|U tan²θ + V tanθ + W + Z/cos²θ|. A negative sum is used as its magnitude without a message. The Technical Reference's formula has no absolute value. + +**PowderLine does:** Documented, not validated. TOPAS carries your value consistently. + +(eb-69)= +#### EB-69 · A negative TCHZ Lorentzian part is computed as written + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** TCHZ_Peak_Type combines a Gaussian width G and a Lorentzian width L = X·tanθ + Y/cosθ into one FWHM. With a fixed X < 0, TOPAS 6 neither takes |X| nor clamps it at 0. X = −0.01 runs, and the profile differs from X = 0. X = −0.1 makes the width negative at the top of the window, and TOPAS writes no results: a negative convolution FWHM stops TOPAS. + +**PowderLine does:** PowderLine honours what TOPAS computes: a negative Lorentzian part passes while the total width is positive. With `TCHZ_Peak_Type`, the total width at the recipe's starting values must be positive at every data point of the fit window. Otherwise validation fails and names X, Y and the 2θ where the width is not positive. An output recipe whose refined values break the rule is not valid. + +**You see:** validation error `topas_tchz_width_not_positive`, `output_recipe_not_valid` + +(eb-56)= +#### EB-56 · PVII_Peak_Type differs from the Technical Reference + +{bdg-secondary}`documented` · TOPAS 7 macro file; the schema targets TOPAS 6 + +**TOPAS does:** TOPAS 7's macro computes h = ha + hb/cosθ + hc·tanθ and m = 0.6 + ma + mb/cosθ + mc·tanθ. The Technical Reference prints the tanθ and 1/cosθ terms the other way round. h1 and h2 are half-widths. + +**PowderLine does:** The schema documents the macro's formula. + +(eb-57)= +#### EB-57 · CS and Strain terms are not sizes or strains + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** TOPAS's own documentation says `CS_L`, `CS_G`, `Strain_L` and `Strain_G` "have no direct physical interpretation and should not be reported". The interpretable quantities are LVol-IB = k / IB, with IB = Voigt_Integral_Breadth_GL(1/CS_G, 1/CS_L), and e0 = Voigt_FWHM_GL(Strain_G, Strain_L)·0.25·π/360. + +**PowderLine does:** PowderLine reports LVol-IB (k = 1) and e0 as derived rows computed by TOPAS. It documents that the four input terms are not sizes or strains. + +(eb-64)= +#### EB-64 · An emission line with lg alone crashes TOPAS 6 + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** A `lam` line with `lg` and no `lh` crashes TOPAS 6. + +**PowderLine does:** The schema requires `lh`, and allows `lg` only with it. + +**You see:** validation error + +### Data and the fit window + +(eb-59)= +#### EB-59 · TOPAS fits the points nearest its limits + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** TOPAS fits the points nearest `start_X` and `finish_X`. + +**PowderLine does:** The gateway hands TOPAS the fit limits moved onto the first and last weighted data points inside the window. TOPAS's `r_wp` then equals core's Rwp. + +(eb-66)= +#### EB-66 · An xo_Is peak loses part of its value at the last fitted point + +{bdg-secondary}`documented` · TOPAS 6 + +**TOPAS does:** For a background or single peak without an axial convolution, TOPAS 6 loses part of the peak's value at the last fitted point, wherever the window ends. LaB6's background peak is 1.35 counts low there, about 4 %. No calculation range, step or `exclude` avoids it. + +**PowderLine does:** Documented, no workaround. The background-only run carries the same value, so `y_bkg` is consistent with `y_calc`. + +**Note:** TOPAS 7 unverified. + +### Refinement, convergence and failures + +(eb-60)= +#### EB-60 · tc.exe gives the same exit code on success and failure + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** `tc.exe` returns the same exit code on success and failure. It prints its messages only to its console window. + +**PowderLine does:** Success is judged from the expected output files, removed before the run. A missing file raises `EngineExecutionError`. Its failure record lists the INP and data kept, the `tc.exe` path and exit code, and whatever `tc.exe` printed. Otherwise look at TOPAS's console window and `tc.log`. + +**You see:** `output_missing` + +(eb-85)= +#### EB-85 · TOPAS 6 exposes no convergence signal + +{bdg-secondary}`documented` · TOPAS 6 + +**TOPAS does:** TOPAS 6 writes no convergence signal in its files. + +**PowderLine does:** `converged` is always `null` for TOPAS 6. + +(eb-63)= +#### EB-63 · A non-finite term silently zeroes a phase + +{bdg-warning}`warned` · TOPAS 6 + +**TOPAS does:** A non-finite term silently zeroes a phase, and the background absorbs the pattern. + +**PowderLine does:** A phase with all-zero calculated intensities gets an informative note. A zero contribution can be legitimate: a phase absent from the sample. + +**You see:** `topas_phase_contributes_nothing` + +(eb-65)= +#### EB-65 · The independent-parameter count works but is undocumented + +{bdg-warning}`warned` · TOPAS 6 + +**TOPAS does:** `Get(number_independent_parameters)` works in TOPAS 6 although it is undocumented. It excludes parameters with zero derivative. + +**PowderLine does:** PowderLine uses it as `parameters_varied`, and warns when it is lower than the parameters requested. + +**You see:** `topas_parameters_not_varied` + +### Platform and installation + +(eb-67)= +#### EB-67 · tc.exe carries no version metadata + +{bdg-success}`handled` · TOPAS 6 + +**TOPAS does:** `tc.exe` carries no version metadata. It prints its version only to its console. + +**PowderLine does:** You declare the installed version (`--topas-version`, or `topas.version` in `.powderline_config.yaml`). A run needs it. The check compares declarations, not the binary. + +(eb-61)= +#### EB-61 · TOPAS 6 cannot output the fit's background + +{bdg-warning}`warned` · TOPAS 6 + +**TOPAS does:** In TOPAS 6, `Get(bkg)` is 0. `phase_out_X` exists in TOPAS 7 only. + +**PowderLine does:** PowderLine runs a second, background-only INP at the refined values. Its calculated pattern, every convolution included, is `y_bkg`. If that run writes nothing or hits the time limit, the result stays a success with an empty `y_bkg` and a warning. The fit's values, statistics and output recipe do not depend on it. + +**You see:** `topas_background_not_calculated` diff --git a/docs/_generated/examples/example_DRX_33.svg b/docs/_generated/examples/example_DRX_33.svg new file mode 100644 index 0000000..0c07737 --- /dev/null +++ b/docs/_generated/examples/example_DRX_33.svg @@ -0,0 +1,29 @@ + + + +example_DRX_33 (GSAS-II): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_DRX_33_anisoADP.svg b/docs/_generated/examples/example_DRX_33_anisoADP.svg new file mode 100644 index 0000000..03c4a9b --- /dev/null +++ b/docs/_generated/examples/example_DRX_33_anisoADP.svg @@ -0,0 +1,29 @@ + + + +example_DRX_33_anisoADP (GSAS-II): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_DRX_33_atomrefine.svg b/docs/_generated/examples/example_DRX_33_atomrefine.svg new file mode 100644 index 0000000..32ccb68 --- /dev/null +++ b/docs/_generated/examples/example_DRX_33_atomrefine.svg @@ -0,0 +1,29 @@ + + + +example_DRX_33_atomrefine (GSAS-II): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_DRX_33_easydiffraction.svg b/docs/_generated/examples/example_DRX_33_easydiffraction.svg new file mode 100644 index 0000000..f9b2a59 --- /dev/null +++ b/docs/_generated/examples/example_DRX_33_easydiffraction.svg @@ -0,0 +1,29 @@ + + + +example_DRX_33 (easydiffraction): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_DRX_33_simulation.svg b/docs/_generated/examples/example_DRX_33_simulation.svg new file mode 100644 index 0000000..4d9dc20 --- /dev/null +++ b/docs/_generated/examples/example_DRX_33_simulation.svg @@ -0,0 +1,29 @@ + + + +example_DRX_33_simulation (GSAS-II): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_DRX_33_strainonly.svg b/docs/_generated/examples/example_DRX_33_strainonly.svg new file mode 100644 index 0000000..7063fb8 --- /dev/null +++ b/docs/_generated/examples/example_DRX_33_strainonly.svg @@ -0,0 +1,29 @@ + + + +example_DRX_33_strainonly (GSAS-II): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_LaB6.svg b/docs/_generated/examples/example_LaB6.svg new file mode 100644 index 0000000..b01ff06 --- /dev/null +++ b/docs/_generated/examples/example_LaB6.svg @@ -0,0 +1,29 @@ + + + +example_LaB6 (GSAS-II): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_LaB6_easydiffraction.svg b/docs/_generated/examples/example_LaB6_easydiffraction.svg new file mode 100644 index 0000000..6ee4a4b --- /dev/null +++ b/docs/_generated/examples/example_LaB6_easydiffraction.svg @@ -0,0 +1,29 @@ + + + +example_LaB6 (easydiffraction): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_LaB6_ionic.svg b/docs/_generated/examples/example_LaB6_ionic.svg new file mode 100644 index 0000000..66cad1e --- /dev/null +++ b/docs/_generated/examples/example_LaB6_ionic.svg @@ -0,0 +1,29 @@ + + + +example_LaB6_ionic (GSAS-II): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_LaB6_ionic_easydiffraction.svg b/docs/_generated/examples/example_LaB6_ionic_easydiffraction.svg new file mode 100644 index 0000000..6e25922 --- /dev/null +++ b/docs/_generated/examples/example_LaB6_ionic_easydiffraction.svg @@ -0,0 +1,29 @@ + + + +example_LaB6_ionic (easydiffraction): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_LaB6_simulation.svg b/docs/_generated/examples/example_LaB6_simulation.svg new file mode 100644 index 0000000..89ae5d7 --- /dev/null +++ b/docs/_generated/examples/example_LaB6_simulation.svg @@ -0,0 +1,29 @@ + + + +example_LaB6_simulation (GSAS-II): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_LaB6_singlepeakfit.svg b/docs/_generated/examples/example_LaB6_singlepeakfit.svg new file mode 100644 index 0000000..2499756 --- /dev/null +++ b/docs/_generated/examples/example_LaB6_singlepeakfit.svg @@ -0,0 +1,29 @@ + + + +example_LaB6_singlepeakfit (GSAS-II): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_LaB6_temperature_series_easydiffraction.svg b/docs/_generated/examples/example_LaB6_temperature_series_easydiffraction.svg new file mode 100644 index 0000000..d18c1a9 --- /dev/null +++ b/docs/_generated/examples/example_LaB6_temperature_series_easydiffraction.svg @@ -0,0 +1,29 @@ + + + +example_LaB6_temperature_series (easydiffraction): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_LaB6_textured.svg b/docs/_generated/examples/example_LaB6_textured.svg new file mode 100644 index 0000000..3ee24bf --- /dev/null +++ b/docs/_generated/examples/example_LaB6_textured.svg @@ -0,0 +1,29 @@ + + + +example_LaB6_textured (GSAS-II): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/examples/example_LaB6_textured_easydiffraction.svg b/docs/_generated/examples/example_LaB6_textured_easydiffraction.svg new file mode 100644 index 0000000..a224d46 --- /dev/null +++ b/docs/_generated/examples/example_LaB6_textured_easydiffraction.svg @@ -0,0 +1,29 @@ + + + +example_LaB6_textured (easydiffraction): observed (grey), calculated (blue) and difference (orange) over the fit window + + + + + + + + + + +2 +4 +6 +8 +10 +12 +14 +2θ (°) +Intensity + + + + + + diff --git a/docs/_generated/ids.md b/docs/_generated/ids.md new file mode 100644 index 0000000..08d10d9 --- /dev/null +++ b/docs/_generated/ids.md @@ -0,0 +1,118 @@ + + +## Engine behaviours + +| ID | Engine | Title | Status | +|---|---|---|---| +| {ref}`EB-01 ` | GSAS-II | GSAS-II uses its own special-position tolerance | handled | +| {ref}`EB-02 ` | GSAS-II | A coordinate next to a cell edge gets a wrong multiplicity | handled | +| {ref}`EB-03 ` | GSAS-II | Origin choice 1 is silently ignored | handled | +| {ref}`EB-04 ` | GSAS-II | A rhombohedral symbol means hexagonal axes unless it ends in R | handled | +| {ref}`EB-05 ` | GSAS-II | Some site symmetries cannot be named, and refining them crashes | handled | +| {ref}`EB-06 ` | GSAS-II | A fixed cell is not checked against its symmetry | handled | +| {ref}`EB-07 ` | GSAS-II | Uij site-symmetry constraints apply only while refining | handled | +| {ref}`EB-08 ` | TOPAS | TOPAS compares positions within 1e-15 | handled | +| {ref}`EB-09 ` | TOPAS | Cell values are used as given | handled | +| {ref}`EB-10 ` | TOPAS | The adps keyword applies the Uij constraints | handled | +| {ref}`EB-11 ` | TOPAS | A bare two-origin symbol is origin choice 1 | handled | +| {ref}`EB-12 ` | easydiffraction | CrysPy rationalizes and snaps coordinates | handled | +| {ref}`EB-13 ` | all engines | gemmi strips the charge from an element symbol | handled | +| {ref}`EB-20 ` | GSAS-II | Any occupancy is accepted | handled | +| {ref}`EB-21 ` | TOPAS | Occupancy is limited per iteration | handled | +| {ref}`EB-22 ` | easydiffraction | Occupancy outside [0, 1] is refused | handled | +| {ref}`EB-23 ` | easydiffraction | CrysFML ignores anisotropic ADPs | handled | +| {ref}`EB-24 ` | TOPAS | TOPAS's isotropic ADP is beq, not Uiso | handled | +| {ref}`EB-25 ` | all engines | Each engine spells ions its own way | handled | +| {ref}`EB-30 ` | GSAS-II | Parameter limits apply only after all cycles | handled | +| {ref}`EB-31 ` | GSAS-II | Cell limits act on the reciprocal metric | handled | +| {ref}`EB-32 ` | GSAS-II | Refinement messages are only printed and stored | warned | +| {ref}`EB-33 ` | GSAS-II | The parameter count includes parameters dropped as singular | warned | +| {ref}`EB-34 ` | GSAS-II | The background-peak width is a variance | handled | +| {ref}`EB-35 ` | GSAS-II | Background peaks are evaluated with silent floors | handled | +| {ref}`EB-36 ` | GSAS-II | A new project adds broadening and background you did not ask for | warned | +| {ref}`EB-37 ` | GSAS-II | SH/L and Peak List widths are floored silently | handled | +| {ref}`EB-38 ` | GSAS-II | A peak outside the fit window makes the peak fit fail with a bare TypeError | handled | +| {ref}`EB-39 ` | GSAS-II | Exported peak widths use other units, and some formulas are wrong | handled | +| {ref}`EB-40 ` | GSAS-II | Some 2-fold sites in R32, R-3m and R-3c are misread | handled | +| {ref}`EB-41 ` | GSAS-II | Some spellings are not found, and two settings are read as another | handled | +| {ref}`EB-42 ` | easydiffraction | A space group is a short name plus a setting code | handled | +| {ref}`EB-43 ` | TOPAS | A bare rhombohedral symbol means hexagonal axes | handled | +| {ref}`EB-44 ` | GSAS-II | GSAS-II rewrites phase names | handled | +| {ref}`EB-45 ` | easydiffraction | Atom labels are restricted and datablock names lowercased | handled | +| {ref}`EB-46 ` | easydiffraction | Negative ADPs are refused at input | handled | +| {ref}`EB-47 ` | easydiffraction | Wyckoff positions are detected only when the structure is built | handled | +| {ref}`EB-48 ` | easydiffraction | An environment variable can turn errors into log messages | handled | +| {ref}`EB-49 ` | TOPAS | Two parameters with the same name are one parameter | handled | +| {ref}`EB-50 ` | all engines | A Uaniso that is not positive definite is used as given | warned | +| {ref}`EB-51 ` | GSAS-II | Negative ADPs are carried and refined | documented | +| {ref}`EB-52 ` | GSAS-II | GOF has two meanings, and R values are capped at 100 % | handled | +| {ref}`EB-53 ` | GSAS-II | Single-peak fitting ignores some refine flags silently | handled | +| {ref}`EB-54 ` | GSAS-II | GSAS-II fits one point past the upper limit | handled | +| {ref}`EB-55 ` | TOPAS | TCHZ takes the absolute value of its Gaussian term | documented | +| {ref}`EB-56 ` | TOPAS | PVII_Peak_Type differs from the Technical Reference | documented | +| {ref}`EB-57 ` | TOPAS | CS and Strain terms are not sizes or strains | handled | +| {ref}`EB-58 ` | TOPAS | Keywords and macros carry default limits | handled | +| {ref}`EB-59 ` | TOPAS | TOPAS fits the points nearest its limits | handled | +| {ref}`EB-60 ` | TOPAS | tc.exe gives the same exit code on success and failure | handled | +| {ref}`EB-61 ` | TOPAS | TOPAS 6 cannot output the fit's background | warned | +| {ref}`EB-62 ` | TOPAS | The limit markers TOPAS writes are unreliable | handled | +| {ref}`EB-63 ` | TOPAS | A non-finite term silently zeroes a phase | warned | +| {ref}`EB-64 ` | TOPAS | An emission line with lg alone crashes TOPAS 6 | handled | +| {ref}`EB-65 ` | TOPAS | The independent-parameter count works but is undocumented | warned | +| {ref}`EB-66 ` | TOPAS | An xo_Is peak loses part of its value at the last fitted point | documented | +| {ref}`EB-67 ` | TOPAS | tc.exe carries no version metadata | handled | +| {ref}`EB-68 ` | TOPAS | Some settings have no TOPAS symbol, and three are misread | handled | +| {ref}`EB-69 ` | TOPAS | A negative TCHZ Lorentzian part is computed as written | handled | +| {ref}`EB-70 ` | easydiffraction | Only the handed points are calculated | documented | +| {ref}`EB-71 ` | easydiffraction | Unbounded sides get physical limits, and starts are moved silently | handled | +| {ref}`EB-72 ` | easydiffraction | The ASCII data reader rounds 2θ to 4 decimals | handled | +| {ref}`EB-73 ` | easydiffraction | A σ below 1e-4 becomes 1.0 silently | handled | +| {ref}`EB-74 ` | easydiffraction | easydiffraction fixes special coordinates and re-detects them on an edit | handled | +| {ref}`EB-75 ` | easydiffraction | The X-ray corrections easydiffraction models, and its polarization | handled | +| {ref}`EB-76 ` | easydiffraction | The two calculators compute different peak types | handled | +| {ref}`EB-77 ` | easydiffraction | CrysPy computes some special positions wrongly | handled | +| {ref}`EB-78 ` | easydiffraction | CrysFML's integrated intensities drift with angle | documented | +| {ref}`EB-79 ` | easydiffraction | CrysFML reports no reflection list | warned | +| {ref}`EB-80 ` | easydiffraction | Each calculator misreads some settings | handled | +| {ref}`EB-81 ` | easydiffraction | A parameter without effect gets no ESD | warned | +| {ref}`EB-82 ` | easydiffraction | Importing easydiffraction queries PyPI | documented | +| {ref}`EB-83 ` | easydiffraction | CrysPy resets the unique angle of some monoclinic cells to 90° | handled | +| {ref}`EB-84 ` | easydiffraction | CrysPy misreads 41 settings and crashes on 53 | handled | +| {ref}`EB-85 ` | TOPAS | TOPAS 6 exposes no convergence signal | documented | +| {ref}`EB-86 ` | easydiffraction | A refined Uij can land in the wrong atom's column | handled | +| {ref}`EB-87 ` | easydiffraction | Negative peak widths give a wrong pattern without a message | handled | +| {ref}`EB-88 ` | all engines | A symmetric profile shifts the refined cell | documented | +| {ref}`EB-89 ` | easydiffraction | The CIF loop cannot read exponent notation | handled | +| {ref}`EB-90 ` | easydiffraction | CrysFML refinements depend on their starting values | warned | +| {ref}`EB-91 ` | easydiffraction | Negative measured intensities are refused | handled | +| {ref}`EB-92 ` | GSAS-II | A peak fit fills no covariance block | handled | +| {ref}`EB-93 ` | easydiffraction | On the temperature series, easydiffraction refines La's Uiso low | documented | +| {ref}`EB-94 ` | GSAS-II | An ion in CIF spelling is read as hydrogen | handled | +| {ref}`EB-95 ` | easydiffraction | An ion CrysPy has no form factor for becomes the neutral atom | handled | +| {ref}`EB-96 ` | easydiffraction | CrysFML computes neutral atoms only | handled | +| {ref}`EB-97 ` | easydiffraction | Preferred orientation reaches CrysPy only, under the right key | handled | +| {ref}`EB-98 ` | easydiffraction | CrysPy's March correction is Dollase's times r^1.5 | documented | +| {ref}`EB-99 ` | GSAS-II | An aborted refinement keeps a worthless pattern | handled | + +## Known issues + +| ID | Applies to | Title | Status | +|---|---|---|---| +| {ref}`KI-01 ` | core | A cell that breaks its space group is refused | resolved | +| {ref}`KI-02 ` | gsasii | A partly refined GSAS-II cell group is refused | resolved | +| {ref}`KI-03 ` | core, gsasii | Site symmetry and multiplicity are derived and checked | resolved | +| {ref}`KI-04 ` | core | Every result carries an output recipe and per-parameter dispositions | resolved | +| {ref}`KI-05 ` | gsasii | GSAS-II refinement failures raise, with the console text attached | resolved | +| {ref}`KI-06 ` | gsasii | GSAS-II recipes have no parameter bounds | accepted | +| {ref}`KI-07 ` | core, gsasii | The space-group setting is stated, never inferred | resolved | +| {ref}`KI-08 ` | docs build | The docs build treats every warning as an error | resolved | +| {ref}`KI-09 ` | — | Withdrawn | withdrawn | +| {ref}`KI-10 ` | GSAS-II server | Stopping an unresponsive GSAS-II server escalates to a forced kill on every platform | resolved | +| {ref}`KI-11 ` | GSAS-II server | A server whose state directory is deleted keeps running unseen | accepted | +| {ref}`KI-12 ` | GSAS-II server | The GSAS-II server returns output files inside its response | resolved | +| {ref}`KI-13 ` | installation | Engine dispatch depends on install-time entry-point metadata | resolved | +| {ref}`KI-14 ` | GSAS-II server | The GSAS-II server is private to the user who started it | resolved | +| {ref}`KI-15 ` | examples | Four example references refine unphysical values, so their output recipes are invalid | accepted | +| {ref}`KI-16 ` | topas | TOPAS execution is verified on TOPAS 6 | resolved | +| {ref}`KI-17 ` | core | A split atom's occupancy is not checked against its coinciding images | accepted | +| {ref}`KI-18 ` | core | A coordinate that refines close to a special position is reported only through the output recipe | accepted | diff --git a/docs/_generated/known_issues.md b/docs/_generated/known_issues.md new file mode 100644 index 0000000..555aed0 --- /dev/null +++ b/docs/_generated/known_issues.md @@ -0,0 +1,169 @@ + + +## Accepted + +(ki-06)= +### KI-06 · GSAS-II recipes have no parameter bounds + +{bdg-warning}`accepted` · minor · gsasii + +**What happens:** GSAS-II enforces a limit by resetting an out-of-range value to the limit after a refine, and freezing it for the next one ({ref}`EB-30 `). Coordinate limits are set on `Ax`, `Ay`, `Az`, while GSAS-II varies `dAx`, `dAy`, `dAz`. Cell limits map exactly onto direct-cell boxes for only four of the seven crystal systems ({ref}`EB-31 `). `gsasii.*` recipes therefore expose no bounds: a 4-element parameter list is a validation error. + +**What to do:** Write `[value, flag]` in `gsasii.*` recipes. `topas.*` and `easydiffraction.*` recipes have bounds, each with its engine's semantics. Bounds may come in a later gsasii schema version. + +(ki-11)= +### KI-11 · A server whose state directory is deleted keeps running unseen + +{bdg-warning}`accepted` · minor · GSAS-II server + +**What happens:** The GSAS-II server keeps its state in a per-user directory: `server.lock`, `server.pid`, and `server.json` with the port and token. "Running" means holding the kernel lock on `server.lock`, and clients use only the endpoint in `server.json`. If that directory is deleted while a server runs, for example `$XDG_RUNTIME_DIR` removed at logout or a temp-directory cleaner, the server keeps running unused. Clients cannot find it, and `gsas-server status` and `stop` cannot see it. A running job's scratch files, under the same directory, are lost too. + +**What to do:** Kill the server by hand: `pkill -u "$USER" -f "gsasii[/.]server"`. + +(ki-15)= +### KI-15 · Four example references refine unphysical values, so their output recipes are invalid + +{bdg-warning}`accepted` · minor · examples + +**What happens:** Four committed GSAS-II references end on values the schemas refuse. `example_DRX_33` refines a Li4MgWO6 crystallite size of about −1355 µm. `example_DRX_33_anisoADP` refines a size of about −9.7 µm. `example_DRX_33_atomrefine` refines a Li occupancy of about −0.35 and a size of about −8 µm. `example_LaB6_singlepeakfit` refines its background peak's σ² to about −2700 centideg², below GSAS-II's silent floor ({ref}`EB-35 `). These runs return `output_recipe_valid: false`, with the warning `output_recipe_not_valid` naming the value. The easydiffraction leg of `example_LaB6_ionic` is in the same position: its fit refines CrysPy's Lorentzian `X` negative ({ref}`EB-87 `). + +**What to do:** Read these as teaching examples of ill-conditioned refinements: the SVD warnings, the railed sizes. `example_DRX_33_strainonly` is the physical DRX_33 variant. Each example's description gives the result of a physical re-posing where one exists. + +(ki-17)= +### KI-17 · A split atom's occupancy is not checked against its coinciding images + +{bdg-warning}`accepted` · minor · core + +**What happens:** An atom placed off a special position to model disorder has `k` symmetry images where the ordered atom had one. `k` is the general multiplicity divided by the special one. Unless its occupancy is divided by `k`, the site holds `k` atoms. PowderLine checks the position, which must be at least 2e-3 off the special position. It does not check the occupancy, because a full occupancy is also a legitimate model: a genuinely general site. + +**What to do:** Divide the occupancy by `k` yourself. See [Model disorder near special positions](../how-to/disorder.md). + +(ki-18)= +### KI-18 · A coordinate that refines close to a special position is reported only through the output recipe + +{bdg-warning}`accepted` · minor · core + +**What happens:** Validation refuses an atom stated 5e-6 to 2e-3 (fractional) from a special position. During a refinement, no engine moves a coordinate back onto one. GSAS-II never snaps, easydiffraction detects Wyckoff positions only when the structure is built ({ref}`EB-47 `), and TOPAS compares exactly. So a split atom can refine into that band. The result is still returned, because its numbers are what the engine computed. Its output recipe fails validation, and the warning `output_recipe_not_valid` names the coordinate, so a chained next step stops there. + +**What to do:** State the atom on the special position, or restart it further off. See [Model disorder near special positions](../how-to/disorder.md). + +## Resolved + +(ki-01)= +### KI-01 · A cell that breaks its space group is refused + +{bdg-success}`resolved` · critical · core + +**What happens:** GSAS-II uses a fixed cell as given, even when it breaks the space group ({ref}`EB-06 `). A cubic phase with a ≠ b ≠ c would give wrong peak positions and a refinement that reports success. Core refuses such a cell in every engine schema. Tied lengths and angles must equal the first member, and fixed angles must be exact, within 1e-9. The error gives the value to write. + +**What to do:** Write the value the error names. PowderLine has no switch that accepts a symmetry-breaking cell. + +(ki-02)= +### KI-02 · A partly refined GSAS-II cell group is refused + +{bdg-success}`resolved` · minor · gsasii + +**What happens:** GSAS-II refines a cell through its reciprocal-metric terms. Holding one of them equals holding a direct-cell parameter only for orthogonal cells. In a b-unique monoclinic cell `a`, `c` and `β` are coupled. In a triclinic cell, and on rhombohedral axes, all six are. A `gsasii.*` recipe that refines a strict subset of such a group, or of a symmetry tie group, is a validation error at the offending field. A flag is never silently overridden. + +**What to do:** Refine the whole group or none of it. `engine_details.parameters_requested` and `parameters_varied` show what GSAS-II varied. + +(ki-03)= +### KI-03 · Site symmetry and multiplicity are derived and checked + +{bdg-success}`resolved` · major · core, gsasii + +**What happens:** Core derives each atom's site symmetry, multiplicity and tie groups with gemmi, without an engine. It cross-checks a stated `Multiplicity`. After phase setup, the gsasii gateway compares GSAS-II's reading of every site with core's. It stops with an error when GSAS-II misreads a site or cannot name it ({ref}`EB-40 `, {ref}`EB-05 `). Nothing is swallowed. + +**What to do:** Site symmetry is derived, not stated. If a GSAS-II run stops with `unusable_site`, write the equivalent position the error names. + +(ki-04)= +### KI-04 · Every result carries an output recipe and per-parameter dispositions + +{bdg-success}`resolved` · major · core + +**What happens:** Every result carries `output_recipe`: the resolved recipe, with the refined values written back at their recipe paths. It also carries `parameter_dispositions`, one row per refinable value, with its ESD and `at_limit`. The disposition is `varied`, `held_by_user`, `held_by_symmetry`, `requested_not_varied` or `default_applied`. When a refined value breaks a schema rule, the output recipe is still returned, with `output_recipe_valid: false` and the warning `output_recipe_not_valid`. + +**What to do:** To chain runs, swap only the `xrd_data` block into each step's output recipe. `example_LaB6_temperature_series` does this with GSAS-II and with easydiffraction. See [Run a sequential refinement](../how-to/sequential-refinement.md). + +(ki-05)= +### KI-05 · GSAS-II refinement failures raise, with the console text attached + +{bdg-success}`resolved` · major · gsasii + +**What happens:** GSAS-II's `refine()` does not raise on failure. Its notices go to the console, or into the project. The gsasii gateway reads the stored messages into `gsasii_refinement_message` warnings ({ref}`EB-32 `). It reads the convergence flag into `converged`, with `gsasii_fit_not_converged`. A non-finite pattern or an aborted refinement raises `EngineExecutionError`, reason `diverged`. The other reasons are `unusable_site`, `peak_fit_failed` and `engine_crashed`. The gateway also keeps a bounded tail (64 KiB) of GSAS-II's Python console output for each run. + +**What to do:** After a failure, read `failure.console_tail`, in the exception's `.result` or in `failure.json`. The CLI prints its last lines after "Refinement failed". A verbose run that succeeded carries the tail under `engine_details`. Output from compiled code is not captured. + +(ki-07)= +### KI-07 · The space-group setting is stated, never inferred + +{bdg-success}`resolved` · critical · core, gsasii + +**What happens:** `R -3 c` on hexagonal axes and `R -3 c R` on rhombohedral axes imply different cells. GSAS-II reads a two-origin group in origin choice 2, so origin-1 coordinates would run on the wrong structure without a message. Core requires gemmi's canonical extended Hermann–Mauguin name, which states the setting: `:1` or `:2`, `:H` or `:R`, and the monoclinic axis. Any other spelling is an error naming the canonical form. A bare two-origin or rhombohedral symbol lists both choices. The gsasii gateway refuses origin choice 1 and 9 further settings ({ref}`EB-03 `, {ref}`EB-41 `). + +**What to do:** Write the canonical name. For a setting the gsasii gateway refuses, transform the structure to a standard setting. + +(ki-08)= +### KI-08 · The docs build treats every warning as an error + +{bdg-success}`resolved` · nit · docs build + +**What happens:** `pixi run docs` runs `sphinx-build -W --keep-going` with `nitpicky = True`. Targets that are documented nowhere Sphinx can link to are listed in `nitpick_ignore` in `docs/conf.py`. + +**What to do:** Keep the build warning-free. Add a target to `nitpick_ignore` only when it has no documentation Sphinx can link to. + +(ki-10)= +### KI-10 · Stopping an unresponsive GSAS-II server escalates to a forced kill on every platform + +{bdg-success}`resolved` · minor · GSAS-II server + +**What happens:** When the server has not exited 5 s after a stop request, `stop` escalates to a forced kill. On Windows, where `SIGKILL` does not exist, it uses `SIGTERM`, which Python maps to `TerminateProcess`, a hard kill. Unit tests drive the escalation path with a mocked process. The Windows branch has not been run against a live, unresponsive server. + +**What to do:** Nothing in normal use. If `stop` fails on Windows, end the server process by hand. + +(ki-12)= +### KI-12 · The GSAS-II server returns output files inside its response + +{bdg-success}`resolved` · major · GSAS-II server + +**What happens:** The server runs each job in a private scratch directory under its per-user state directory. It returns every PowderLine output file, base64-encoded, inside the signed response. The client writes the files into its own output directory and lists them in `output_files`. No shared filesystem is needed, so a server on another node or in a container works like a local one. The response is a versioned envelope. + +**What to do:** Nothing. + +(ki-13)= +### KI-13 · Engine dispatch depends on install-time entry-point metadata + +{bdg-success}`resolved` · minor · installation + +**What happens:** `powderline.run()` and `powderline.validate()` find every gateway through the `powderline.gateways` entry points of the installed distribution. That is the only discovery path. If an environment's editable install is stale or half-installed, or PowderLine is imported from source with `PYTHONPATH` and no installed distribution, every run fails with `GatewayNotInstalledError`. The message names the fix. + +**What to do:** Run `pixi run powderline --doctor`. It lists the gateways the registry finds and each engine's version against its requirement. It exits non-zero, naming the repair, when something is missing. After editing `[project.entry-points]` or `[project.scripts]`, run `pixi reinstall powderline --all`. If pixi cannot rebuild the editable install, use that environment's own Python: `python -m pip install --no-deps --no-build-isolation --no-index -e .`. + +(ki-14)= +### KI-14 · The GSAS-II server is private to the user who started it + +{bdg-success}`resolved` · critical · GSAS-II server + +**What happens:** Each user gets their own server. Its lock, PID, log and endpoint live in a private per-user state directory, with mode 0600 files on POSIX. The server listens on `127.0.0.1` on an ephemeral port. Every request and every response is signed with a per-start token that never crosses the wire. Unsigned, wrongly signed, stale or replayed requests are refused. A kernel-held lock on `server.lock` allows one server per user. The request carries no output directory: files travel back inside the response ({ref}`KI-12 `). + +**What to do:** Nothing in normal use. To keep any server from starting, set `POWDERLINE_NO_SERVER=1`. Then `auto` runs in-process, `server` mode and `--use-server` fail, and `gsas-server start` refuses. Operation is described in [Run at a beamline](../how-to/beamline-operation.md). + +(ki-16)= +### KI-16 · TOPAS execution is verified on TOPAS 6 + +{bdg-success}`resolved` · major · topas + +**What happens:** The `topas_exec` tests run `tc.exe` against the committed goldens under `tests/data/topas/native/golden/6/`. On the release's code they reproduced all seven golden fixtures file for file through `powderline.run()`. The ionic example's La3+/La intensity ratios agree with GSAS-II's within 0.069 %. The textured example recovers the March–Dollase ratio as 0.5982(14), against the programmed 0.6. Without `tc.exe`, a run is generate-only, with the goldens as the contract. + +**What to do:** On a TOPAS machine, run `pixi run test-topas` with `POWDERLINE_TOPAS_DIR` and `POWDERLINE_TOPAS_VERSION` set. TOPAS 7 is not supported: it needs its own goldens and the same tests. + +## Withdrawn + +(ki-09)= +### KI-09 · Withdrawn + +{bdg-secondary}`withdrawn` + +**What happens:** The reported behaviour was intentional, not a defect. + +**What to do:** Nothing. The id is kept so it is never reused. diff --git a/docs/catalog/engine_behaviour.yaml b/docs/catalog/engine_behaviour.yaml new file mode 100644 index 0000000..8373c5e --- /dev/null +++ b/docs/catalog/engine_behaviour.yaml @@ -0,0 +1,1439 @@ +# PowderLine engine-behaviour catalogue (public). +# +# One entry per EB-NN id: what an engine does on its own, and what PowderLine +# does about it. scripts/gen_catalog.py renders it into docs/_generated/ +# (eb_.md, ids.md), which the Engines pages include; `--check` fails +# when an EB id used in src/ or docs/ has no entry here. +# +# Public fields only. Never write evidence paths, probe names, source line +# numbers or private record ids here. +# +# Fields +# id EB-NN, permanent, never reused. +# engine gsasii | topas | easydiffraction | all +# (all: concerns every engine, or compares engines). +# versions the engine version(s) the entry was established on. +# topic one of the topics below. +# title one short sentence: what the engine does. +# engine_does plain, short sentences. +# powderline_does plain, short sentences. +# you_see warning codes, failure reasons or "validation error"; +# may be empty. A bare code is rendered in backticks. +# status one of the statuses below. +# seen_in optional: example names where it shows. +# note optional: a caveat (unverified, shared id, ...). +# +# Topics (rendered in this order, headings in sentence case) +# space-groups Space groups +# sites Special positions and sites +# cell-and-adps Cell, ADPs and occupancy +# scattering Scattering types +# names Names +# bounds Bounds and limits +# defaults Defaults +# profile Peak profile and instrument +# preferred-orientation Preferred orientation +# data Data and the fit window +# refinement Refinement, convergence and failures +# statistics Statistics +# calculators Calculators +# platform Platform and installation +# +# Statuses +# handled PowderLine prevents or corrects it before or during the +# run: a validation error, a translation, or a check that +# stops the run. A recipe cannot meet it silently. +# warned The run goes ahead and the result carries a warning (or a +# field) that names it. +# documented Documented only: the engine's behaviour stands, and the +# entry tells you what to expect. +# fixed-upstream Fixed in the engine version PowderLine pins; the entry +# stays for its id. + +# --- Space groups ----------------------------------------------------------- + +- id: EB-03 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: space-groups + title: Origin choice 1 is silently ignored + engine_does: >- + GSAS-II's own space-group lookup tolerates `:H` and `:2` and silently + ignores `:1`. A recipe meant in origin choice 1 runs in origin choice 2, + on a different structure. + powderline_does: >- + The gsasii gateway refuses every origin-choice-1 setting (33 settings). + To use one, transform the structure to a standard setting. + you_see: [validation error] + status: handled + +- id: EB-04 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: space-groups + title: A rhombohedral symbol means hexagonal axes unless it ends in R + engine_does: >- + GSAS-II reads `R -3 m` and `R -3 m:H` on hexagonal axes. `R -3 m R` or + `:R` gives rhombohedral axes. + powderline_does: >- + Core requires the canonical name, which states `:H` or `:R` explicitly. + A bare rhombohedral symbol is an error that lists both choices. The gsasii + gateway writes a trailing ` R` for `:R` and drops `:H`. + you_see: [validation error] + status: handled + +- id: EB-41 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: space-groups + title: Some spellings are not found, and two settings are read as another + engine_does: >- + GSAS-II reads spaced symbols. Its lookup returns nothing for spellings + such as `R-3m:H` or `Fd-3m:2`. It reads `P 21 n m` and `A b a m` as + another setting, whose general position has a different orbit. + powderline_does: >- + The gsasii gateway translates the canonical name: a spaced symbol, ` R` + for `:R`, `:H` and `:2` dropped. It refuses 9 settings beyond origin + choice 1: 7 that GSAS-II does not parse and 2 that it misreads. A + general-position orbit test over every accepted setting proves the + translation. + you_see: [validation error] + status: handled + +- id: EB-11 + engine: topas + versions: TOPAS 6 + topic: space-groups + title: A bare two-origin symbol is origin choice 1 + engine_does: >- + TOPAS accepts the symbols of its own table: case-insensitive, spaces + allowed, or the number. `:2` selects origin choice 2. A bare two-origin + symbol is origin choice 1, the opposite of GSAS-II's default. + powderline_does: >- + The topas gateway writes TOPAS's explicit form: `:2` kept, `:1` dropped. + you_see: [] + status: handled + +- id: EB-43 + engine: topas + versions: TOPAS 6 + topic: space-groups + title: A bare rhombohedral symbol means hexagonal axes + engine_does: >- + TOPAS reads a bare rhombohedral symbol on hexagonal axes. A trailing `r` + gives rhombohedral axes. + powderline_does: >- + The topas gateway drops `:H` and writes `:R` as a trailing `r`. + you_see: [] + status: handled + +- id: EB-68 + engine: topas + versions: TOPAS 6 + topic: space-groups + title: Some settings have no TOPAS symbol, and three are misread + engine_does: >- + Twenty of gemmi's settings have no TOPAS symbol. TOPAS 6 reads + `C 1 21 1`, `I 1 21 1` and `C 4 2 21` as another setting, dropping an + origin shift. + powderline_does: >- + The topas gateway refuses these 23 settings. The other 541 were checked + operator by operator against TOPAS 6's own generated operator files. + you_see: [validation error] + status: handled + +- id: EB-42 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: space-groups + title: A space group is a short name plus a setting code + engine_does: >- + easydiffraction takes one of 230 short names and a separate setting code: + 531 settings in all. + powderline_does: >- + The easydiffraction gateway translates the canonical name into a short + name and a setting code. It refuses what easydiffraction cannot express. + you_see: [validation error] + status: handled + +- id: EB-80 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + topic: space-groups + title: Each calculator misreads some settings + engine_does: >- + CrysFML is handed the short symbol alone. It misreads almost every + non-default setting: unique axis, cell choice, `:R` and some origin-1 + groups. CrysPy computes `P 31` with the operators of `P 32`. + powderline_does: >- + Validation accepts, per calculator, only the settings whose computation + passed for a general position and for every special site: 428 of the 531 + for CrysPy, 230 for CrysFML. The error names a setting of the same group + that the calculator computes, or the other calculator. + you_see: [validation error] + status: handled + +- id: EB-84 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysPy 0.13.0) + topic: space-groups + title: CrysPy misreads 41 settings and crashes on 53 + engine_does: >- + CrysPy computes 41 settings as another setting, and crashes on 53. + powderline_does: >- + Validation accepts only the settings CrysPy computes correctly (428 of + 531). The error names a setting of the same group that CrysPy computes, + or the crysfml calculator. + you_see: [validation error] + status: handled + note: >- + The public account cites EB-84 after both facts in one sentence; whether + the 41 misread settings belong to EB-84 or EB-80 is not stated + separately. + +- id: EB-83 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysPy 0.13.0) + topic: space-groups + title: CrysPy resets the unique angle of some monoclinic cells to 90° + engine_does: >- + For c-unique and a-unique monoclinic cells, CrysPy resets the unique + angle to 90°. + powderline_does: >- + These settings are not among those validation accepts for the cryspy + calculator. The error names a setting of the same group that CrysPy + computes, or the crysfml calculator. + you_see: [validation error] + status: handled + +# --- Special positions and sites -------------------------------------------- + +- id: EB-01 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: sites + title: GSAS-II uses its own special-position tolerance + engine_does: >- + GSAS-II compares symmetry images within 2e-5 + 1e-5·|x|, and not + modulo 1. + powderline_does: >- + Core decides special positions, not the engine. An atom within 5e-6 + (fractional) of a special position is on it. An atom 5e-6 to 2e-3 off is + a validation error. Derived values are held exactly in the validated + model, so GSAS-II's tolerance never matters. + you_see: [symmetry_value_derived, validation error] + status: handled + +- id: EB-02 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: sites + title: A coordinate next to a cell edge gets a wrong multiplicity + engine_does: >- + A coordinate just below an integer gets multiplicity 0. A coordinate just + above 0 gets an invalid multiplicity. + powderline_does: >- + This cannot be reached: such coordinates are exact constants in the + validated model. + you_see: [] + status: handled + +- id: EB-40 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: sites + title: Some 2-fold sites in R32, R-3m and R-3c are misread + engine_does: >- + On hexagonal axes, GSAS-II reads some orientations of these 2-fold sites, + such as (0, 1/3, 1/6), as a general position with doubled multiplicity. + With nothing refined, it computes four times the Bragg intensity of the + equivalent (1/3, 0, 1/6). + powderline_does: >- + After phase setup, the gsasii gateway compares GSAS-II's site symmetry and + multiplicity with core's for every atom. A wrong multiplicity always stops + the run, even in a simulation. The error names an equivalent position + GSAS-II reads correctly. PowderLine never substitutes the position + silently, because refined coordinates would then differ from the recipe. + you_see: [unusable_site] + status: handled + +- id: EB-05 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: sites + title: Some site symmetries cannot be named, and refining them crashes + engine_does: >- + GSAS-II cannot name the symmetry of some site orientations (`'sp'`), for + example F432 at (0, y, 0). It crashes when a coordinate or `Uaniso` of + such an atom is refined. + powderline_does: >- + After phase setup, the gsasii gateway stops with an error naming an + equivalent position GSAS-II can name. It does so only when a coordinate + or `Uaniso` of that atom is refined. + you_see: [unusable_site] + status: handled + +- id: EB-08 + engine: topas + versions: TOPAS 6 + topic: sites + title: TOPAS compares positions within 1e-15 + engine_does: >- + TOPAS compares equivalent positions within 1e-15. It treats any decimal + third as a general position. + powderline_does: >- + Core holds derived values exactly. The topas writer emits fixed special + coordinates as exact equations, such as `= 1/3;`. + you_see: [] + status: handled + +- id: EB-12 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysPy 0.13.0) + topic: sites + title: CrysPy rationalizes and snaps coordinates + engine_does: >- + CrysPy rationalizes each coordinate to a denominator of at most 10⁵. It + snaps 5-decimal thirds. + powderline_does: >- + Core hands over exact values, so CrysPy's own tolerance never matters. + you_see: [] + status: handled + +- id: EB-47 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: sites + title: Wyckoff positions are detected only when the structure is built + engine_does: >- + easydiffraction applies site symmetry itself. It detects each atom's + Wyckoff position when the structure is built, and never re-detects during + a fit. + powderline_does: >- + The gateway hands over core's exact values. After the build, it checks + easydiffraction's multiplicity against core's for every atom. A refined + coordinate that ends near a special position is reported through the + output recipe (KI-18). + you_see: [output_recipe_not_valid] + status: handled + +- id: EB-74 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: sites + title: easydiffraction fixes special coordinates and re-detects them on an edit + engine_does: >- + easydiffraction marks fixed coordinates and ignores a `free` flag on + them. It re-detects the Wyckoff position on a user edit with tolerance + 1e-3, so an off-special coordinate can change the multiplicity. + powderline_does: >- + The gateway hands over core's exact values. After the build, it checks + easydiffraction's multiplicity against core's for every atom. + you_see: [] + status: handled + note: >- + The public account cites EB-47 and EB-74 together for easydiffraction's + site-symmetry handling; the part that is EB-74's own is not stated + separately. + +- id: EB-77 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysPy 0.13.0) + topic: sites + title: CrysPy computes some special positions wrongly + engine_does: >- + CrysPy computes wrong structure factors for some equivalent writings of a + special position with a free coordinate. LaB6's B written as (½, ½, x) + instead of the template's (x, ½, ½) gives intensities off by up to ×2. + It also computes special positions wrongly in nine origin-choice-1 + settings. General positions and fixed special positions are right. + powderline_does: >- + With `calculator: "cryspy"`, an atom on such a site must be written in the + pattern of easydiffraction's Wyckoff template. The builder refuses + anything else before the run and names the position to write. A position + that easydiffraction only stores translated by a lattice vector is the + same atom: it is accepted with a warning, and reports keep the recipe's + writing. The nine settings are not accepted for the cryspy calculator. + CrysFML has no such rule. + you_see: [validation error, easydiffraction_position_translated] + status: handled + +# --- Cell, ADPs and occupancy ----------------------------------------------- + +- id: EB-06 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: cell-and-adps + title: A fixed cell is not checked against its symmetry + engine_does: >- + GSAS-II enforces cell symmetry only while the cell is refined. A fixed, + inconsistent cell is used as given and gives wrong peak positions. + powderline_does: >- + Core refuses a cell that does not fit the space group, within 1e-9, in + every engine. The error gives the value to write (KI-01). + you_see: [validation error] + status: handled + +- id: EB-09 + engine: topas + versions: TOPAS 6 + topic: cell-and-adps + title: Cell values are used as given + engine_does: >- + TOPAS uses the individual `a b c al be ga` values as given. + powderline_does: >- + Core refuses a cell that does not fit the space group, within 1e-9, in + every engine. + you_see: [validation error] + status: handled + +- id: EB-07 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: cell-and-adps + title: Uij site-symmetry constraints apply only while refining + engine_does: >- + GSAS-II applies the site-symmetry constraints on Uij when refining. It + stores the values as given. + powderline_does: >- + Core requires `Uij` values that respect the site symmetry, before any + engine runs. + you_see: [validation error] + status: handled + +- id: EB-10 + engine: topas + versions: TOPAS 6 + topic: cell-and-adps + title: The adps keyword applies the Uij constraints + engine_does: >- + TOPAS's `adps` keyword applies the site-symmetry constraints on Uij. + powderline_does: >- + Core requires `Uij` values that respect the site symmetry, before any + engine runs. + you_see: [validation error] + status: handled + note: Unverified. Recorded from documentation, not from a run. + +- id: EB-24 + engine: topas + versions: TOPAS 6 + topic: cell-and-adps + title: TOPAS's isotropic ADP is beq, not Uiso + engine_does: >- + TOPAS states the isotropic ADP as `beq` = 8π²·Uiso. + powderline_does: >- + The recipe states `Uiso` in Ų, as in every schema. The topas gateway + converts the value and its bounds to `beq` and warns once per atom. + Results report `Uiso`. + you_see: [topas_adp_converted] + status: handled + +- id: EB-23 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysFML 0.8.0) + topic: cell-and-adps + title: CrysFML ignores anisotropic ADPs + engine_does: >- + easydiffraction supports anisotropic ADPs, but CrysFML is handed only + B = 8π²·Ueq. A `Uaniso` has no effect with CrysFML. + powderline_does: >- + `Uaniso` with the crysfml calculator is a validation error. + you_see: [validation error] + status: handled + +- id: EB-86 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysPy 0.13.0) + topic: cell-and-adps + title: A refined Uij can land in the wrong atom's column + engine_does: >- + With CrysPy, easydiffraction writes a refined Uij into the wrong atom's + column whenever an isotropic atom precedes the anisotropic one. The + refined tensor then has no effect. + powderline_does: >- + The gateway hands the anisotropic atoms to easydiffraction first. Reports + keep the recipe's order. A test proves that a refined Uij moves. + you_see: [] + status: handled + +- id: EB-51 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: cell-and-adps + title: Negative ADPs are carried and refined + engine_does: >- + GSAS-II carries negative `Uiso` and `Uij` values and refines from them. + powderline_does: >- + Core has no sign rule for ADPs: a negative ADP can compensate for an + unmodelled effect. + you_see: [] + status: documented + +- id: EB-50 + engine: all + versions: GSAS-II 5.7.9; easydiffraction 0.21.1; TOPAS 6 unverified + topic: cell-and-adps + title: A Uaniso that is not positive definite is used as given + engine_does: >- + GSAS-II has no positive-definiteness check. easydiffraction computes a + finite pattern for such a tensor, and for a negative ADP. TOPAS is + unverified. + powderline_does: >- + Core uses the tensor as stated and warns when it has a principal value + ≤ 0. + you_see: [uaniso_not_positive_definite] + status: warned + note: >- + The public account cites EB-46 and EB-50 together for easydiffraction's + finite pattern. + +- id: EB-46 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: cell-and-adps + title: Negative ADPs are refused at input + engine_does: >- + easydiffraction refuses a negative `Uiso` or U11, U22, U33 at input. Its + physical limit for them is 0–10 Ų. + powderline_does: >- + An `easydiffraction.*` recipe may not state a negative `Uiso` or U11, + U22, U33: validation error. + you_see: [validation error] + status: handled + +- id: EB-20 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: cell-and-adps + title: Any occupancy is accepted + engine_does: >- + GSAS-II accepts any occupancy value. + powderline_does: >- + Core requires 0 ≤ occupancy ≤ 1 for every engine. + you_see: [validation error] + status: handled + +- id: EB-21 + engine: topas + versions: TOPAS 6 + topic: cell-and-adps + title: Occupancy is limited per iteration + engine_does: >- + TOPAS limits `occ` to 0 … 2·value + 1 per iteration. + powderline_does: >- + Core requires 0 ≤ occupancy ≤ 1 for every engine. + you_see: [validation error] + status: handled + +- id: EB-22 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: cell-and-adps + title: Occupancy outside [0, 1] is refused + engine_does: >- + easydiffraction refuses an occupancy outside [0, 1]. + powderline_does: >- + Core requires 0 ≤ occupancy ≤ 1 for every engine, so the recipe is + refused before easydiffraction sees it. + you_see: [validation error] + status: handled + +# --- Scattering types --------------------------------------------------------- + +- id: EB-13 + engine: all + versions: gemmi 0.7.5 + topic: scattering + title: gemmi strips the charge from an element symbol + engine_does: >- + gemmi, which core uses for symmetry and elements, strips the charge from + a symbol: `Fe3+` becomes Fe. + powderline_does: >- + `element` takes a bare element symbol only. An ion in `element` is + refused, never stripped. Ions are stated in `scattering_types`, in the CIF + spelling. + you_see: [validation error] + status: handled + +- id: EB-25 + engine: all + versions: GSAS-II 5.7.9; TOPAS 6; easydiffraction 0.21.1 (CrysPy 0.13.0) + topic: scattering + title: Each engine spells ions its own way + engine_does: >- + GSAS-II and TOPAS spell an ion `Fe+3`. CrysPy spells it `Fe3+`, the CIF + spelling. + powderline_does: >- + You write every ion in the CIF spelling, in `scattering_types`. Each + gateway writes its engine's spelling. `Fe+3` in a recipe is an error that + names `Fe3+`. + you_see: [validation error] + status: handled + note: >- + The public account cites EB-13 and EB-25 together for ion notation; this + entry holds the spelling difference between engines. + +- id: EB-94 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: scattering + title: An ion in CIF spelling is read as hydrogen + engine_does: >- + GSAS-II reads the CIF spelling `La3+` as hydrogen (Z = 1), silently. Its + own spelling is `La+3`. Its form-factor table has 114 ions. + powderline_does: >- + You write `La3+` in `scattering_types`. The gsasii gateway writes + GSAS-II's spelling into the atom record. It refuses an ion GSAS-II has no + form factor for. + you_see: [validation error] + status: handled + seen_in: [example_LaB6_ionic] + +- id: EB-95 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysPy 0.13.0) + topic: scattering + title: An ion CrysPy has no form factor for becomes the neutral atom + engine_does: >- + easydiffraction accepts any CIF type symbol, such as `O2-`. When CrysPy + has no form factor for the ion, it uses the neutral atom with a log + warning only. CrysPy has 111 ions: `O1-` but no `O2-`. + powderline_does: >- + An ion outside CrysPy's table is a validation error that names the forms + CrysPy has. + you_see: [validation error] + status: handled + +- id: EB-96 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysFML 0.8.0) + topic: scattering + title: CrysFML computes neutral atoms only + engine_does: >- + easydiffraction strips every charge before handing the type to CrysFML. + powderline_does: >- + `scattering_types` with the crysfml calculator is a validation error. Use + the cryspy calculator for ions. + you_see: [validation error] + status: handled + +# --- Names ------------------------------------------------------------------ + +- id: EB-44 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: names + title: GSAS-II rewrites phase names + engine_does: >- + GSAS-II strips surrounding whitespace from a phase name and replaces + non-ASCII characters with `_`. The settings of a renamed phase are then + skipped silently. A `/` in the name breaks the report file write. + powderline_does: >- + The core name rule makes every such name invalid before an engine runs: + `^[A-Za-z][A-Za-z0-9_]*$`, at most 64 characters, no Python keyword, + distinct ignoring case. + you_see: [validation error] + status: handled + +- id: EB-45 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: names + title: Atom labels are restricted and datablock names lowercased + engine_does: >- + easydiffraction accepts atom labels of the form + `[A-Za-z_][A-Za-z0-9_]*` only. It lowercases datablock names. + powderline_does: >- + The core name rule makes every name easydiffraction would reject or merge + invalid before it runs. + you_see: [validation error] + status: handled + +- id: EB-49 + engine: topas + versions: TOPAS 6 + topic: names + title: Two parameters with the same name are one parameter + engine_does: >- + TOPAS treats two `prm` of the same name as one parameter, so composed + names could collide silently. + powderline_does: >- + The topas writer uses index-based parameter names. The core name rule + also applies. + you_see: [] + status: handled + note: Unverified beyond TOPAS's documented model. + +# --- Bounds and limits ------------------------------------------------------ + +- id: EB-30 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: bounds + title: Parameter limits apply only after all cycles + engine_does: >- + GSAS-II applies parameter limits only after all cycles of a `Refine` + call. An out-of-range value is set to the limit with ESD 0 and frozen for + the next call. The reported values, ESDs and Rwp belong to the unclamped + fit. + powderline_does: >- + `gsasii.*` recipes expose no bounds. A 4-element parameter list is a + validation error (KI-06). + you_see: [validation error] + status: handled + +- id: EB-31 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: bounds + title: Cell limits act on the reciprocal metric + engine_does: >- + Cell limits must be set on the reciprocal-metric terms. They map exactly + onto direct-cell boxes for only four of the seven crystal systems. + powderline_does: >- + `gsasii.*` recipes expose no bounds. A 4-element parameter list is a + validation error. + you_see: [validation error] + status: handled + +- id: EB-58 + engine: topas + versions: TOPAS 6 + topic: bounds + title: Keywords and macros carry default limits + engine_does: >- + TOPAS keywords carry default limits: `scale` ≥ 1e-11, `beq` in [−10, 20], + cell lengths ≥ 1.5 Å, lattice ±0.5 % per iteration, among others. The + TOPAS.INC macros add their own: TCHZ U–Z in [−1, 2], X and Y in + [1e-4, 2], CS in [0.3, 10000] nm, Strain in [1e-4, 5], and the + March–Dollase `PO` ratio min 1e-4 and max 2r + 0.5. A user `min` or `max` + overrides the default on that side. A start outside a limit is moved onto + it silently. + powderline_does: >- + A stated bound replaces TOPAS's limit on that side. A `null` side keeps + TOPAS's default limit. A refined start outside a fixed default limit on a + `null` side is a validation error: state a bound, or start inside. A + refined value within 1e-4 (relative) of a stated bound or a default limit + gets a warning. At a limit of 0 the tolerance is 1e-12 absolute. + you_see: [validation error, topas_parameter_at_limit] + status: handled + +- id: EB-62 + engine: topas + versions: TOPAS 6 + topic: bounds + title: The limit markers TOPAS writes are unreliable + engine_does: >- + The `_LIMIT_MIN` and `_LIMIT_MAX` markers TOPAS 6 writes are mislabelled + or spurious. + powderline_does: >- + The at-limit warning is judged from the final values and the limits the + writer knows, not from TOPAS's markers. + you_see: [topas_parameter_at_limit] + status: handled + +- id: EB-71 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: bounds + title: Unbounded sides get physical limits, and starts are moved silently + engine_does: >- + When a fit starts, easydiffraction gives an unbounded side its physical + limit: cell lengths 0–30 Å, angles 0–180°, occupancy 0–1, `adp_iso` and + U11/U22/U33 0–10 Ų, scale, wavelength and μR ≥ 0. It refuses a value + outside a physical limit at input, even when fixed, so no cell over 30 Å + can be modelled. lmfit moves a start outside its bounds onto the bound + without a message. + powderline_does: >- + A stated bound replaces the physical limit on its side, and `null` keeps + it. A value outside a physical limit is a validation error. A refined + value that ends within 1e-6 (relative) of a bound or physical limit gets + a warning. For a limit of 0, the test is within 1e-6 of the starting + value. + you_see: [validation error, easydiffraction_parameter_at_limit] + status: handled + +# --- Defaults ----------------------------------------------------------------- + +- id: EB-36 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: defaults + title: A new project adds broadening and background you did not ask for + engine_does: >- + A new GSAS-II project gets isotropic size 1 µm, microstrain 1000 and a + constant 1.0 Chebyshev background, all fixed. + powderline_does: >- + Size and strain left out mean 10 µm and 0 microstrain, isotropic, fixed: + no sample broadening. A background left out means none: one fixed + Chebyshev term 0.0. Each default comes with a warning naming the phase + and the values. State the blocks to silence the warning. + you_see: [gsasii_broadening_default_applied, gsasii_background_default_applied] + status: warned + +# --- Peak profile and instrument -------------------------------------------- + +- id: EB-37 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: profile + title: SH/L and Peak List widths are floored silently + engine_does: >- + GSAS-II floors SH/L at 0.002 in the Rietveld and Peak List profile + calculations. It floors it at 0.0005 in `GetFobsSq`, and background peaks + use it as stored. Peak List peaks floor σ² and γ at 0.001. + powderline_does: >- + `axial_divergence` below 0.002 is a validation error. So is a Peak List + width below its floor. The converted LaB6 examples state 0.002 where they + said 0.0005. + you_see: [validation error] + status: handled + +- id: EB-35 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: profile + title: Background peaks are evaluated with silent floors + engine_does: >- + GSAS-II evaluates background peaks with floors: intensity 0.1, σ² + 0.01 centideg², γ 0.1 centideg. + powderline_does: >- + A stated value below a floor is a validation error. A refined value below + a floor makes the output recipe invalid (KI-15). + you_see: [validation error, output_recipe_not_valid] + status: handled + seen_in: [example_LaB6_singlepeakfit] + +- id: EB-34 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: profile + title: The background-peak width is a variance + engine_does: >- + GSAS-II's `BkPksig` is σ² in centideg², although its GUI calls it `sig` + and its prose calls it "Gaussian width". + powderline_does: >- + The recipe field is named `pv_gaussian_sigma_sq`. + you_see: [] + status: handled + +- id: EB-39 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: profile + title: Exported peak widths use other units, and some formulas are wrong + engine_does: >- + GSAS-II's peak `int` is an integrated area. Its exporters write σ in + degrees while the project holds σ² in centideg², and several exporter + formulas are wrong. + powderline_does: >- + PowderLine reads the project directly and uses none of those exporters. + you_see: [] + status: handled + +- id: EB-55 + engine: topas + versions: TOPAS 6 + topic: profile + title: TCHZ takes the absolute value of its Gaussian term + engine_does: >- + TCHZ computes ΓG = √|U tan²θ + V tanθ + W + Z/cos²θ|. A negative sum is + used as its magnitude without a message. The Technical Reference's + formula has no absolute value. + powderline_does: >- + Documented, not validated. TOPAS carries your value consistently. + you_see: [] + status: documented + +- id: EB-69 + engine: topas + versions: TOPAS 6 + topic: profile + title: A negative TCHZ Lorentzian part is computed as written + engine_does: >- + TCHZ_Peak_Type combines a Gaussian width G and a Lorentzian width + L = X·tanθ + Y/cosθ into one FWHM. With a fixed X < 0, TOPAS 6 neither + takes |X| nor clamps it at 0. X = −0.01 runs, and the profile differs + from X = 0. X = −0.1 makes the width negative at the top of the window, + and TOPAS writes no results: a negative convolution FWHM stops TOPAS. + powderline_does: >- + PowderLine honours what TOPAS computes: a negative Lorentzian part passes + while the total width is positive. With `TCHZ_Peak_Type`, the total width + at the recipe's starting values must be positive at every data point of + the fit window. Otherwise validation fails and names X, Y and the 2θ where + the width is not positive. An output recipe whose refined values break the + rule is not valid. + you_see: [validation error `topas_tchz_width_not_positive`, output_recipe_not_valid] + status: handled + +- id: EB-56 + engine: topas + versions: TOPAS 7 macro file; the schema targets TOPAS 6 + topic: profile + title: PVII_Peak_Type differs from the Technical Reference + engine_does: >- + TOPAS 7's macro computes h = ha + hb/cosθ + hc·tanθ and + m = 0.6 + ma + mb/cosθ + mc·tanθ. The Technical Reference prints the + tanθ and 1/cosθ terms the other way round. h1 and h2 are half-widths. + powderline_does: >- + The schema documents the macro's formula. + you_see: [] + status: documented + +- id: EB-57 + engine: topas + versions: TOPAS 6 + topic: profile + title: CS and Strain terms are not sizes or strains + engine_does: >- + TOPAS's own documentation says `CS_L`, `CS_G`, `Strain_L` and `Strain_G` + "have no direct physical interpretation and should not be reported". The + interpretable quantities are LVol-IB = k / IB, with + IB = Voigt_Integral_Breadth_GL(1/CS_G, 1/CS_L), and + e0 = Voigt_FWHM_GL(Strain_G, Strain_L)·0.25·π/360. + powderline_does: >- + PowderLine reports LVol-IB (k = 1) and e0 as derived rows computed by + TOPAS. It documents that the four input terms are not sizes or strains. + you_see: [] + status: handled + +- id: EB-64 + engine: topas + versions: TOPAS 6 + topic: profile + title: An emission line with lg alone crashes TOPAS 6 + engine_does: >- + A `lam` line with `lg` and no `lh` crashes TOPAS 6. + powderline_does: >- + The schema requires `lh`, and allows `lg` only with it. + you_see: [validation error] + status: handled + +- id: EB-75 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + topic: profile + title: The X-ray corrections easydiffraction models, and its polarization + engine_does: >- + Both calculators model the Kα doublet, the Lorentz-polarization factor + 1 − p + p cos²2θ_m cos²2θ, the 2θ offset, sample displacement and + transparency, and Hewat cylinder absorption. A second wavelength alone + does nothing. 2θ_m acts only with p > 0. GSAS-II's polarization P is + easydiffraction's 1 − p: copying P to p is a 6.6 % intensity error at + 15° 2θ. + powderline_does: >- + Every correction is a recipe slot; left out means not modelled. The + schema requires both Kα2 fields or neither. The hand-derived per-engine + recipes state p = 1 − P. + you_see: [validation error] + status: handled + +- id: EB-87 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + topic: profile + title: Negative peak widths give a wrong pattern without a message + engine_does: >- + Where U tan²θ + V tanθ + W < 0, CrysFML drops the peaks and CrysPy + distorts them. Where X tanθ + Y/cosθ < 0, both give a finite pattern + 12–24 % off. + powderline_does: >- + A stated profile that makes a width negative is a validation error. The + check covers the handed points and a 0.01° grid over the window widened + by 1° plus the position corrections. A refined profile that does so gets + a warning. The LaB6 CrysPy pseudo-Voigt fit refines X to −0.098, + negative from 11.2° on. + you_see: [validation error, easydiffraction_negative_width] + status: handled + seen_in: [example_LaB6_ionic] + +# --- Preferred orientation -------------------------------------------------- + +- id: EB-97 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + topic: preferred-orientation + title: Preferred orientation reaches CrysPy only, under the right key + engine_does: >- + Only the CrysPy calculator reads the experiment's preferred-orientation + rows. CrysFML's pattern does not change with `march_r`, without a + message. CrysPy also ignores a row whose `structure_id` is not the + lowercase linked structure name, silently. + powderline_does: >- + `preferred_orientation` with `calculator: "crysfml"` is a validation + error. The builder keys the row by the structure name easydiffraction + uses. + you_see: [validation error] + status: handled + +- id: EB-98 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysPy 0.13.0) + topic: preferred-orientation + title: CrysPy's March correction is Dollase's times r^1.5 + engine_does: >- + CrysPy's texture correction is not volume-normalised, so every + textured-to-untextured intensity ratio carries the constant r^1.5. With + `march_random_fract` = 0 the scale absorbs it, and the refined `march_r` + is the same r as GSAS-II's. With f > 0 the correction is + (1 − f)·r^1.5·A + f, and the refined r and f are not GSAS-II's + quantities. + powderline_does: >- + Documented. A test compares normalised ratios with GSAS-II's + March–Dollase correction. + you_see: [] + status: documented + +# --- Data and the fit window ------------------------------------------------ + +- id: EB-54 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: data + title: GSAS-II fits one point past the upper limit + engine_does: >- + When the upper limit falls between data points, GSAS-II fits and scores + one point past it. With no limit, it starts at the first data point, + weighted or not. Its `Nobs` counts the extra point. + powderline_does: >- + The gateway hands GSAS-II the fit limits moved onto the first and last + weighted data points inside the window. GSAS-II then fits exactly the + points core scores, and its own Rwp equals core's. + you_see: [] + status: handled + +- id: EB-59 + engine: topas + versions: TOPAS 6 + topic: data + title: TOPAS fits the points nearest its limits + engine_does: >- + TOPAS fits the points nearest `start_X` and `finish_X`. + powderline_does: >- + The gateway hands TOPAS the fit limits moved onto the first and last + weighted data points inside the window. TOPAS's `r_wp` then equals core's + Rwp. + you_see: [] + status: handled + +- id: EB-70 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: data + title: Only the handed points are calculated + engine_does: >- + easydiffraction calculates only the points it is handed, and maps its + Chebyshev basis over them. Reflection tails from beyond the window are + lost: at most 3e-4 σ at the refined scale. + powderline_does: >- + The gateway hands easydiffraction exactly the weighted points inside the + window, so it fits the points core scores. The lost tails are documented. + you_see: [] + status: documented + +- id: EB-66 + engine: topas + versions: TOPAS 6 + topic: data + title: An xo_Is peak loses part of its value at the last fitted point + engine_does: >- + For a background or single peak without an axial convolution, TOPAS 6 + loses part of the peak's value at the last fitted point, wherever the + window ends. LaB6's background peak is 1.35 counts low there, about 4 %. + No calculation range, step or `exclude` avoids it. + powderline_does: >- + Documented, no workaround. The background-only run carries the same + value, so `y_bkg` is consistent with `y_calc`. + you_see: [] + status: documented + note: TOPAS 7 unverified. + +- id: EB-73 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: data + title: A σ below 1e-4 becomes 1.0 silently + engine_does: >- + On every input path, easydiffraction replaces σ < 1e-4 with σ = 1.0. + powderline_does: >- + Weights above 1e8 on handed points are a validation error. + you_see: [validation error] + status: handled + +- id: EB-91 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: data + title: Negative measured intensities are refused + engine_does: >- + easydiffraction refuses negative measured intensities. + powderline_does: >- + A negative intensity on a handed point is a validation error. + you_see: [validation error] + status: handled + +- id: EB-72 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: data + title: The ASCII data reader rounds 2θ to 4 decimals + engine_does: >- + easydiffraction's ASCII data reader rounds 2θ to 4 decimals. + powderline_does: >- + The gateway hands the data as a CIF loop at full precision. + you_see: [] + status: handled + +- id: EB-89 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: data + title: The CIF loop cannot read exponent notation + engine_does: >- + easydiffraction's CIF loop reader cannot read numbers in exponent + notation. + powderline_does: >- + The gateway writes every number in the data loop in positional notation. + you_see: [] + status: handled + +# --- Refinement, convergence and failures ----------------------------------- + +- id: EB-32 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: refinement + title: Refinement messages are only printed and stored + engine_does: >- + `Refine` returns nothing on success. Its messages about frozen limits, + parameters dropped as singular and SVD are only printed and stored in the + project. + powderline_does: >- + The gateway reads the stored messages into warnings. + you_see: [gsasii_refinement_message] + status: warned + +- id: EB-33 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: refinement + title: The parameter count includes parameters dropped as singular + engine_does: >- + GSAS-II's `Nvars` includes parameters it dropped as singular. + powderline_does: >- + PowderLine reports the drop as a warning and does not subtract it. In a + single-peak fit, the count is taken after the drops. + you_see: [] + status: warned + +- id: EB-53 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: refinement + title: Single-peak fitting ignores some refine flags silently + engine_does: >- + With the instrument profile in use, a peak's own σ² and γ flags are + ignored. With per-peak widths, the U … Z flags are ignored. Refining any + σ² clears the U, V, W flags. Singular parameters are dropped without a + message. + powderline_does: >- + A refine flag on the width source not in use is a validation error. Any + remaining silent drop gets a warning. + you_see: [validation error, gsasii_parameters_not_varied] + status: handled + +- id: EB-92 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: refinement + title: A peak fit fills no covariance block + engine_does: >- + `DoPeakFit` keeps its varied names and ESDs in the Peak List's own + record, and writes the refined values in place. A peak fit therefore has + no refined-parameter rows. + powderline_does: >- + The result's `peak_fit` block reads the values and ESDs from there. The + output recipe is built from it. + you_see: [] + status: handled + +- id: EB-38 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: refinement + title: A peak outside the fit window makes the peak fit fail with a bare TypeError + engine_does: >- + A Peak List or background peak outside the fit window makes GSAS-II's + peak fit fail with a bare `TypeError`. + powderline_does: >- + A peak outside the fit window is a validation error. A peak fit that still + fails raises `EngineExecutionError` with a clear reason. + you_see: [validation error, peak_fit_failed] + status: handled + +- id: EB-99 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: refinement + title: An aborted refinement keeps a worthless pattern + engine_does: >- + When no damped step lowers χ², GSAS-II aborts the refinement. It keeps + the last accepted values but stores the rejected step's pattern and a wR + at its 100 % cap. No statistic belongs to the parameters. GSAS-II's GUI + shows this only as "Minimizer halted because chi**2 increased". A + March–Dollase ratio started at exactly 1.0 in a cubic phase triggers it. + powderline_does: >- + PowderLine is deliberately stricter: an aborted Rietveld refinement is a + failure, reason `diverged`. Simulations are exempt. + you_see: [diverged] + status: handled + +- id: EB-60 + engine: topas + versions: TOPAS 6 + topic: refinement + title: tc.exe gives the same exit code on success and failure + engine_does: >- + `tc.exe` returns the same exit code on success and failure. It prints its + messages only to its console window. + powderline_does: >- + Success is judged from the expected output files, removed before the run. + A missing file raises `EngineExecutionError`. Its failure record lists + the INP and data kept, the `tc.exe` path and exit code, and whatever + `tc.exe` printed. Otherwise look at TOPAS's console window and `tc.log`. + you_see: [output_missing] + status: handled + +- id: EB-85 + engine: topas + versions: TOPAS 6 + topic: refinement + title: TOPAS 6 exposes no convergence signal + engine_does: >- + TOPAS 6 writes no convergence signal in its files. + powderline_does: >- + `converged` is always `null` for TOPAS 6. + you_see: [] + status: documented + +- id: EB-63 + engine: topas + versions: TOPAS 6 + topic: refinement + title: A non-finite term silently zeroes a phase + engine_does: >- + A non-finite term silently zeroes a phase, and the background absorbs the + pattern. + powderline_does: >- + A phase with all-zero calculated intensities gets an informative note. A + zero contribution can be legitimate: a phase absent from the sample. + you_see: [topas_phase_contributes_nothing] + status: warned + +- id: EB-65 + engine: topas + versions: TOPAS 6 + topic: refinement + title: The independent-parameter count works but is undocumented + engine_does: >- + `Get(number_independent_parameters)` works in TOPAS 6 although it is + undocumented. It excludes parameters with zero derivative. + powderline_does: >- + PowderLine uses it as `parameters_varied`, and warns when it is lower than + the parameters requested. + you_see: [topas_parameters_not_varied] + status: warned + +# --- Statistics ------------------------------------------------------------- + +- id: EB-52 + engine: gsasii + versions: GSAS-II 5.7.9 + topic: statistics + title: GOF has two meanings, and R values are capped at 100 % + engine_does: >- + GSAS-II's `GOF` is √(reduced χ²) in a Rietveld refinement and the reduced + χ² itself in a Peak List fit. GSAS-II caps R and wR at 100 %, so a + simulation against placeholder data and a diverged refinement both read + 100 %. A non-finite pattern prints wR `nan` and reports 100 %. + powderline_does: >- + The result's `rwp`, `r_exp`, `gof` and `chi2_red` are PowderLine's, with + one formula for every engine. GSAS-II's own values stay under + `engine_details`. A non-finite calculated pattern is a failure, reason + `diverged`. A poor fit at the cap is still reported, with core's uncapped + Rwp beside it. + you_see: [diverged] + status: handled + +- id: EB-81 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: statistics + title: A parameter without effect gets no ESD + engine_does: >- + easydiffraction's free-parameter count is lmfit's. A parameter without + effect gets no ESD. Its own `prof_wr_factor` is 1/σ²-weighted. + powderline_does: >- + `parameters_varied` is lmfit's count. The engine's own statistics stay + under `engine_details`. + you_see: [easydiffraction_parameters_not_varied] + status: warned + +- id: EB-88 + engine: all + versions: GSAS-II 5.7.9; easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + topic: statistics + title: A symmetric profile shifts the refined cell + engine_does: >- + Lattice parameters agree between GSAS-II and easydiffraction when axial + asymmetry is modelled physically (FCJ). With a symmetric profile they + shift by +67 to +91 ppm, whichever calculator runs it. + powderline_does: >- + Documented on the cross-engine comparison. Rwp compares models fitted to + the same points, not engines. + you_see: [] + status: documented + +# --- Calculators ------------------------------------------------------------ + +- id: EB-76 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + topic: calculators + title: The two calculators compute different peak types + engine_does: >- + CrysPy computes `cwl-pseudo-voigt` and the Bérar–Baldinozzi empirical + asymmetry. CrysFML computes `cwl-pseudo-voigt` and the + Thompson–Cox–Hastings profile with FCJ axial asymmetry. Both + pseudo-Voigts are the TCH pseudo-Voigt with Caglioti Gaussian and + X tanθ + Y/cosθ Lorentzian widths. `cutoff_fwhm` acts in CrysPy only; + CrysFML uses a fixed 30-FWHM window. + powderline_does: >- + The peak type must be one the chosen calculator computes. Otherwise + validation fails. + you_see: [validation error] + status: handled + +- id: EB-79 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysFML 0.8.0) + topic: calculators + title: CrysFML reports no reflection list + engine_does: >- + CrysFML reports no reflection list. + powderline_does: >- + No peak list reports are written with CrysFML, and the result says so. + you_see: [easydiffraction_reflections_not_available] + status: warned + +- id: EB-90 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysFML 0.8.0) + topic: calculators + title: CrysFML refinements depend on their starting values + engine_does: >- + CrysFML computes in single precision, below lmfit's finite-difference + step. A CrysFML refinement stops at a point that depends on its starting + values. Starts changed by 1e-6 (relative) gave Rwp 6.79 % against 6.27 %, + with cells stable to about 1e-5. + powderline_does: >- + Every CrysFML refinement carries a warning. The calculator stays + available because you choose it, and PowderLine states the difference. + you_see: [easydiffraction_crysfml_start_dependent] + status: warned + +- id: EB-78 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysFML 0.8.0) + topic: calculators + title: CrysFML's integrated intensities drift with angle + engine_does: >- + Relative to m·|F|²·Lorentz, CrysFML's integrated intensities drift by + −4 % from 3° to 15° 2θ. This biases `Uiso` by about +0.001 Ų. + powderline_does: >- + The calculator is recorded in every result. + you_see: [] + status: documented + +- id: EB-93 + engine: easydiffraction + versions: easydiffraction 0.21.1 (CrysPy 0.13.0, CrysFML 0.8.0) + topic: calculators + title: On the temperature series, easydiffraction refines La's Uiso low + engine_does: >- + On GSAS-II-calculated LaB6 patterns, easydiffraction refines La's Uiso + 10–22 % low. B's Uiso, the thermal expansion and GoF ≈ 1 agree. The cause + is the series' profile: GSAS-II clamps its Lorentzian width to + 0.001 centideg above about 10.7°, so the high-angle peaks are pure + Gaussian. Neither a symmetric pseudo-Voigt nor TCH can represent that, + and `Uiso` absorbs the mismatch. Simulated with a profile easydiffraction + can represent, CrysPy's La Uiso is 0.00863 against 0.00858 programmed. + CrysFML's is 0.00823 (−4 %), the size of its intensity drift (EB-78). + powderline_does: >- + Documented, no code change. Across engines, compare cells, with the + EB-88-size offsets in mind, and the expansion. Absolute `Uiso` agrees only + when the profile models agree. + you_see: [] + status: documented + seen_in: [example_LaB6_temperature_series] + +# --- Platform and installation ---------------------------------------------- + +- id: EB-67 + engine: topas + versions: TOPAS 6 + topic: platform + title: tc.exe carries no version metadata + engine_does: >- + `tc.exe` carries no version metadata. It prints its version only to its + console. + powderline_does: >- + You declare the installed version (`--topas-version`, or `topas.version` + in `.powderline_config.yaml`). A run needs it. The check compares + declarations, not the binary. + you_see: [] + status: handled + +- id: EB-61 + engine: topas + versions: TOPAS 6 + topic: platform + title: TOPAS 6 cannot output the fit's background + engine_does: >- + In TOPAS 6, `Get(bkg)` is 0. `phase_out_X` exists in TOPAS 7 only. + powderline_does: >- + PowderLine runs a second, background-only INP at the refined values. Its + calculated pattern, every convolution included, is `y_bkg`. If that run + writes nothing or hits the time limit, the result stays a success with an + empty `y_bkg` and a warning. The fit's values, statistics and output + recipe do not depend on it. + you_see: [topas_background_not_calculated] + status: warned + +- id: EB-48 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: platform + title: An environment variable can turn errors into log messages + engine_does: >- + With `ED_LOG_REACTION=WARN` in the environment, easydiffraction downgrades + its errors to log messages. + powderline_does: >- + The gateway sets the reaction to `RAISE` itself, whatever the + environment. + you_see: [] + status: handled + +- id: EB-82 + engine: easydiffraction + versions: easydiffraction 0.21.1 + topic: platform + title: Importing easydiffraction queries PyPI + engine_does: >- + Importing easydiffraction imports `diffpy.pdffit2`, whose version module + queries PyPI. Offline, it prints a warning and continues. + powderline_does: >- + Nothing. The import, and the run, continue. + you_see: [] + status: documented diff --git a/docs/catalog/known_issues.yaml b/docs/catalog/known_issues.yaml new file mode 100644 index 0000000..fccc6a6 --- /dev/null +++ b/docs/catalog/known_issues.yaml @@ -0,0 +1,342 @@ +# PowderLine known-issues catalogue (public). +# +# One entry per KI-NN id: a known behaviour of PowderLine itself, what it +# does today and what to do about it. scripts/gen_catalog.py renders it into +# docs/_generated/known_issues.md (included by development/known-issues.md) +# and docs/_generated/ids.md; `--check` fails when a KI id used in src/ or +# docs/ has no entry here. +# +# Public facts and the current state only. Never write process here: review +# rounds, dates, evidence file:line lists or private record ids. +# +# Fields +# id KI-NN, permanent, never reused (a withdrawn id stays listed). +# title one short sentence, in the current state. +# status one of the statuses below. +# severity one of the severities below (may be left out when withdrawn). +# what_happens plain, short sentences. +# what_to_do plain, short sentences. +# applies_to optional: the engine or area, e.g. core, gsasii, topas, +# easydiffraction, GSAS-II server, examples, installation, +# docs build. +# +# Statuses +# open known and not yet addressed. +# accepted known, and deliberately not prevented; documented here. +# watch a quirk to keep in mind; no action planned. +# resolved addressed in the current release; the id stays listed. +# withdrawn not a defect; the id stays listed so it is never reused. +# +# Severities (impact if you meet it, independent of status) +# critical silent wrong or empty output, or data loss. +# major breaks a real workflow. +# minor works, but fragile or wrong in edge cases. +# nit cleanup. + +- id: KI-01 + title: A cell that breaks its space group is refused + status: resolved + severity: critical + applies_to: core + what_happens: >- + GSAS-II uses a fixed cell as given, even when it breaks the space group + (EB-06). A cubic phase with a ≠ b ≠ c would give wrong peak positions and + a refinement that reports success. Core refuses such a cell in every + engine schema. Tied lengths and angles must equal the first member, and + fixed angles must be exact, within 1e-9. The error gives the value to + write. + what_to_do: >- + Write the value the error names. PowderLine has no switch that accepts a + symmetry-breaking cell. + +- id: KI-02 + title: A partly refined GSAS-II cell group is refused + status: resolved + severity: minor + applies_to: gsasii + what_happens: >- + GSAS-II refines a cell through its reciprocal-metric terms. Holding one + of them equals holding a direct-cell parameter only for orthogonal cells. + In a b-unique monoclinic cell `a`, `c` and `β` are coupled. In a + triclinic cell, and on rhombohedral axes, all six are. A `gsasii.*` + recipe that refines a strict subset of such a group, or of a symmetry tie + group, is a validation error at the offending field. A flag is never + silently overridden. + what_to_do: >- + Refine the whole group or none of it. `engine_details.parameters_requested` + and `parameters_varied` show what GSAS-II varied. + +- id: KI-03 + title: Site symmetry and multiplicity are derived and checked + status: resolved + severity: major + applies_to: core, gsasii + what_happens: >- + Core derives each atom's site symmetry, multiplicity and tie groups with + gemmi, without an engine. It cross-checks a stated `Multiplicity`. After + phase setup, the gsasii gateway compares GSAS-II's reading of every site + with core's. It stops with an error when GSAS-II misreads a site or + cannot name it (EB-40, EB-05). Nothing is swallowed. + what_to_do: >- + Site symmetry is derived, not stated. If a GSAS-II run stops with + `unusable_site`, write the equivalent position the error names. + +- id: KI-04 + title: Every result carries an output recipe and per-parameter dispositions + status: resolved + severity: major + applies_to: core + what_happens: >- + Every result carries `output_recipe`: the resolved recipe, with the + refined values written back at their recipe paths. It also carries + `parameter_dispositions`, one row per refinable value, with its ESD and + `at_limit`. The disposition is `varied`, `held_by_user`, + `held_by_symmetry`, `requested_not_varied` or `default_applied`. When a + refined value breaks a schema rule, the output recipe is still returned, + with `output_recipe_valid: false` and the warning + `output_recipe_not_valid`. + what_to_do: >- + To chain runs, swap only the `xrd_data` block into each step's output + recipe. `example_LaB6_temperature_series` does this with GSAS-II and with + easydiffraction. See [Run a sequential + refinement](../how-to/sequential-refinement.md). + +- id: KI-05 + title: GSAS-II refinement failures raise, with the console text attached + status: resolved + severity: major + applies_to: gsasii + what_happens: >- + GSAS-II's `refine()` does not raise on failure. Its notices go to the + console, or into the project. The gsasii gateway reads the stored + messages into `gsasii_refinement_message` warnings (EB-32). It reads the + convergence flag into `converged`, with `gsasii_fit_not_converged`. A + non-finite pattern or an aborted refinement raises + `EngineExecutionError`, reason `diverged`. The other reasons are + `unusable_site`, `peak_fit_failed` and `engine_crashed`. The gateway also + keeps a bounded tail (64 KiB) of GSAS-II's Python console output for each + run. + what_to_do: >- + After a failure, read `failure.console_tail`, in the exception's + `.result` or in `failure.json`. The CLI prints its last lines after + "Refinement failed". A verbose run that succeeded carries the tail under + `engine_details`. Output from compiled code is not captured. + +- id: KI-06 + title: GSAS-II recipes have no parameter bounds + status: accepted + severity: minor + applies_to: gsasii + what_happens: >- + GSAS-II enforces a limit by resetting an out-of-range value to the limit + after a refine, and freezing it for the next one (EB-30). Coordinate + limits are set on `Ax`, `Ay`, `Az`, while GSAS-II varies `dAx`, `dAy`, + `dAz`. Cell limits map exactly onto direct-cell boxes for only four of + the seven crystal systems (EB-31). `gsasii.*` recipes therefore expose no + bounds: a 4-element parameter list is a validation error. + what_to_do: >- + Write `[value, flag]` in `gsasii.*` recipes. `topas.*` and + `easydiffraction.*` recipes have bounds, each with its engine's + semantics. Bounds may come in a later gsasii schema version. + +- id: KI-07 + title: The space-group setting is stated, never inferred + status: resolved + severity: critical + applies_to: core, gsasii + what_happens: >- + `R -3 c` on hexagonal axes and `R -3 c R` on rhombohedral axes imply + different cells. GSAS-II reads a two-origin group in origin choice 2, so + origin-1 coordinates would run on the wrong structure without a message. + Core requires gemmi's canonical extended Hermann–Mauguin name, which + states the setting: `:1` or `:2`, `:H` or `:R`, and the monoclinic axis. + Any other spelling is an error naming the canonical form. A bare + two-origin or rhombohedral symbol lists both choices. The gsasii gateway + refuses origin choice 1 and 9 further settings (EB-03, EB-41). + what_to_do: >- + Write the canonical name. For a setting the gsasii gateway refuses, + transform the structure to a standard setting. + +- id: KI-08 + title: The docs build treats every warning as an error + status: resolved + severity: nit + applies_to: docs build + what_happens: >- + `pixi run docs` runs `sphinx-build -W --keep-going` with + `nitpicky = True`. Targets that are documented nowhere Sphinx can link + to are listed in `nitpick_ignore` in `docs/conf.py`. + what_to_do: >- + Keep the build warning-free. Add a target to `nitpick_ignore` only when + it has no documentation Sphinx can link to. + +- id: KI-09 + title: Withdrawn + status: withdrawn + what_happens: >- + The reported behaviour was intentional, not a defect. + what_to_do: >- + Nothing. The id is kept so it is never reused. + +- id: KI-10 + title: Stopping an unresponsive GSAS-II server escalates to a forced kill on every platform + status: resolved + severity: minor + applies_to: GSAS-II server + what_happens: >- + When the server has not exited 5 s after a stop request, `stop` + escalates to a forced kill. On Windows, where `SIGKILL` does not exist, + it uses `SIGTERM`, which Python maps to `TerminateProcess`, a hard kill. + Unit tests drive the escalation path with a mocked process. The Windows + branch has not been run against a live, unresponsive server. + what_to_do: >- + Nothing in normal use. If `stop` fails on Windows, end the server + process by hand. + +- id: KI-11 + title: A server whose state directory is deleted keeps running unseen + status: accepted + severity: minor + applies_to: GSAS-II server + what_happens: >- + The GSAS-II server keeps its state in a per-user directory: `server.lock`, + `server.pid`, and `server.json` with the port and token. "Running" means + holding the kernel lock on `server.lock`, and clients use only the + endpoint in `server.json`. If that directory is deleted while a server + runs, for example `$XDG_RUNTIME_DIR` removed at logout or a temp-directory + cleaner, the server keeps running unused. Clients cannot find it, + and `gsas-server status` and `stop` cannot see it. A running job's + scratch files, under the same directory, are lost too. + what_to_do: >- + Kill the server by hand: `pkill -u "$USER" -f "gsasii[/.]server"`. + +- id: KI-12 + title: The GSAS-II server returns output files inside its response + status: resolved + severity: major + applies_to: GSAS-II server + what_happens: >- + The server runs each job in a private scratch directory under its + per-user state directory. It returns every PowderLine output file, + base64-encoded, inside the signed response. The client writes the files + into its own output directory and lists them in `output_files`. No + shared filesystem is needed, so a server on another node or in a + container works like a local one. The response is a versioned envelope. + what_to_do: >- + Nothing. + +- id: KI-13 + title: Engine dispatch depends on install-time entry-point metadata + status: resolved + severity: minor + applies_to: installation + what_happens: >- + `powderline.run()` and `powderline.validate()` find every gateway through + the `powderline.gateways` entry points of the installed distribution. + That is the only discovery path. If an environment's editable install is + stale or half-installed, or PowderLine is imported from source with + `PYTHONPATH` and no installed distribution, every run fails with + `GatewayNotInstalledError`. The message names the fix. + what_to_do: >- + Run `pixi run powderline --doctor`. It lists the gateways the registry + finds and each engine's version against its requirement. It exits + non-zero, naming the repair, when something is missing. After editing + `[project.entry-points]` or `[project.scripts]`, run + `pixi reinstall powderline --all`. If pixi cannot rebuild the editable + install, use that environment's own Python: + `python -m pip install --no-deps --no-build-isolation --no-index -e .`. + +- id: KI-14 + title: The GSAS-II server is private to the user who started it + status: resolved + severity: critical + applies_to: GSAS-II server + what_happens: >- + Each user gets their own server. Its lock, PID, log and endpoint live in + a private per-user state directory, with mode 0600 files on POSIX. The + server listens on `127.0.0.1` on an ephemeral port. Every request and + every response is signed with a per-start token that never crosses the + wire. Unsigned, wrongly signed, stale or replayed requests are refused. A + kernel-held lock on `server.lock` allows one server per user. The request + carries no output directory: files travel back inside the response + (KI-12). + what_to_do: >- + Nothing in normal use. To keep any server from starting, set + `POWDERLINE_NO_SERVER=1`. Then `auto` runs in-process, `server` mode and + `--use-server` fail, and `gsas-server start` refuses. Operation is + described in [Run at a beamline](../how-to/beamline-operation.md). + +- id: KI-15 + title: Four example references refine unphysical values, so their output recipes are invalid + status: accepted + severity: minor + applies_to: examples + what_happens: >- + Four committed GSAS-II references end on values the schemas refuse. + `example_DRX_33` refines a Li4MgWO6 crystallite size of about −1355 µm. + `example_DRX_33_anisoADP` refines a size of about −9.7 µm. + `example_DRX_33_atomrefine` refines a Li occupancy of about −0.35 and a + size of about −8 µm. `example_LaB6_singlepeakfit` refines its background + peak's σ² to about −2700 centideg², below GSAS-II's silent floor + (EB-35). These runs return `output_recipe_valid: false`, with the warning + `output_recipe_not_valid` naming the value. The easydiffraction leg of + `example_LaB6_ionic` is in the same position: its fit refines CrysPy's + Lorentzian `X` negative (EB-87). + what_to_do: >- + Read these as teaching examples of ill-conditioned refinements: the SVD + warnings, the railed sizes. `example_DRX_33_strainonly` is the physical + DRX_33 variant. Each example's description gives the result of a + physical re-posing where one exists. + +- id: KI-16 + title: TOPAS execution is verified on TOPAS 6 + status: resolved + severity: major + applies_to: topas + what_happens: >- + The `topas_exec` tests run `tc.exe` against the committed goldens under + `tests/data/topas/native/golden/6/`. On the release's code they + reproduced all seven golden fixtures file for file through + `powderline.run()`. The ionic example's La3+/La intensity ratios agree + with GSAS-II's within 0.069 %. The textured example recovers the + March–Dollase ratio as 0.5982(14), against the programmed 0.6. Without + `tc.exe`, a run is generate-only, with the goldens as the contract. + what_to_do: >- + On a TOPAS machine, run `pixi run test-topas` with `POWDERLINE_TOPAS_DIR` + and `POWDERLINE_TOPAS_VERSION` set. TOPAS 7 is not supported: it needs + its own goldens and the same tests. + +- id: KI-17 + title: A split atom's occupancy is not checked against its coinciding images + status: accepted + severity: minor + applies_to: core + what_happens: >- + An atom placed off a special position to model disorder has `k` symmetry + images where the ordered atom had one. `k` is the general multiplicity + divided by the special one. Unless its occupancy is divided by `k`, the + site holds `k` atoms. PowderLine checks the position, which must be at + least 2e-3 off the special position. It does not check the occupancy, + because a full occupancy is also a legitimate model: a genuinely general + site. + what_to_do: >- + Divide the occupancy by `k` yourself. See [Model disorder near special + positions](../how-to/disorder.md). + +- id: KI-18 + title: A coordinate that refines close to a special position is reported only through the output recipe + status: accepted + severity: minor + applies_to: core + what_happens: >- + Validation refuses an atom stated 5e-6 to 2e-3 (fractional) from a + special position. During a refinement, no engine moves a coordinate back + onto one. GSAS-II never snaps, easydiffraction detects Wyckoff positions + only when the structure is built (EB-47), and TOPAS compares exactly. So + a split atom can refine into that band. The result is still returned, + because its numbers are what the engine computed. Its output recipe fails + validation, and the warning `output_recipe_not_valid` names the + coordinate, so a chained next step stops there. + what_to_do: >- + State the atom on the special position, or restart it further off. See + [Model disorder near special positions](../how-to/disorder.md). diff --git a/pixi.toml b/pixi.toml index 84a4f01..5bee12f 100644 --- a/pixi.toml +++ b/pixi.toml @@ -56,6 +56,11 @@ test = { cmd = 'pytest -m "core or topas_gen"', env = { PYTHONPATH = "$PIXI_PROJ # installation, so they run only on the TOPAS Windows machine. The core guard is off here. test-topas = { cmd = 'pytest -m "topas_gen or topas_exec"', env = { PYTHONPATH = "$PIXI_PROJECT_ROOT/src" } } fields-check = { cmd = "python scripts/gen_schema_field_tables.py --check", env = { PYTHONPATH = "$PIXI_PROJECT_ROOT/src" } } +# The EB/KI catalogue fragments (docs/_generated/eb_*.md, ids.md, known_issues.md) are +# current, and every EB-/KI- id cited under src/ and docs/ has a catalogue entry. +catalog-check = { cmd = "python scripts/gen_catalog.py --check", env = { PYTHONPATH = "$PIXI_PROJECT_ROOT/src" } } +# The example fit figures (docs/_generated/examples/*.svg) match the committed fit profiles. +figures-check = { cmd = "python scripts/gen_example_figures.py --check" } [dependencies] # Core = the pyproject runtime dependencies + what the editable install and the diff --git a/scripts/gen_catalog.py b/scripts/gen_catalog.py new file mode 100644 index 0000000..582acbb --- /dev/null +++ b/scripts/gen_catalog.py @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +"""Generate the docs' engine-behaviour entries, ID index and known issues from the public catalogue. + +Every ``EB-NN`` (engine behaviour) and ``KI-NN`` (known issue) id that PowderLine +prints, or that the docs cite, must be look-up-able. The public fields of each +entry live in two data files: + +* ``docs/catalog/engine_behaviour.yaml`` -- one entry per ``EB-NN``; +* ``docs/catalog/known_issues.yaml`` -- one entry per ``KI-NN``. + +This script validates both and writes MyST fragments that the pages pull in +with ``{include}``: + +* ``docs/_generated/eb_.md`` (``gsasii``, ``topas``, + ``easydiffraction``, ``all``) -- the entries of one engine, grouped by topic, + each with a label ``(eb-nn)=`` so ``{ref}`EB-87 ``` links from any page; +* ``docs/_generated/ids.md`` -- the ID index: every EB, then every KI; +* ``docs/_generated/known_issues.md`` -- the KI entries, grouped by status, + each with a label ``(ki-nn)=``. + +Engine-free and PowderLine-free (needs only PyYAML):: + + python scripts/gen_catalog.py # rewrite the fragments + python scripts/gen_catalog.py --check # exit 1 if they are stale, or an id has no entry + +``--check`` also scans every text file under ``src/`` and ``docs/`` (except +``docs/_generated/``, ``docs/catalog/`` and build directories) for ``EB-`` +and ``KI-`` and fails naming any id the catalogue does not hold. + +Edit the YAML, run the script, and commit the YAML and the fragments together. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +CATALOG_DIR = ROOT / "docs" / "catalog" +OUT_DIR = ROOT / "docs" / "_generated" +EB_FILE = CATALOG_DIR / "engine_behaviour.yaml" +KI_FILE = CATALOG_DIR / "known_issues.yaml" +#: Trees scanned by ``--check`` for ids, and the paths under them that are skipped. +SCAN_ROOTS = (ROOT / "src", ROOT / "docs") +SCAN_SKIP = (OUT_DIR, CATALOG_DIR) +SCAN_SKIP_DIRS = {"_build", "__pycache__", ".ipynb_checkpoints"} + +HEADER = "" +ID_RE = re.compile(r"\b(EB|KI)-(\d+)\b") + +# --- vocabularies -------------------------------------------------------------------------------- + +ENGINES = {"gsasii": "GSAS-II", "topas": "TOPAS", "easydiffraction": "easydiffraction", "all": "all engines"} +#: Topic -> heading (sentence case), in rendering order. +TOPICS = { + "space-groups": "Space groups", + "sites": "Special positions and sites", + "cell-and-adps": "Cell, ADPs and occupancy", + "scattering": "Scattering types", + "names": "Names", + "bounds": "Bounds and limits", + "defaults": "Defaults", + "profile": "Peak profile and instrument", + "preferred-orientation": "Preferred orientation", + "data": "Data and the fit window", + "refinement": "Refinement, convergence and failures", + "statistics": "Statistics", + "calculators": "Calculators", + "platform": "Platform and installation", +} +#: EB status -> (badge role, badge text). +EB_STATUSES = { + "handled": ("bdg-success", "handled"), + "warned": ("bdg-warning", "warned"), + "documented": ("bdg-secondary", "documented"), + "fixed-upstream": ("bdg-info", "fixed upstream"), +} +#: KI status -> (badge role, section heading), in rendering order. +KI_STATUSES = { + "open": ("bdg-danger", "Open"), + "accepted": ("bdg-warning", "Accepted"), + "watch": ("bdg-info", "Watch"), + "resolved": ("bdg-success", "Resolved"), + "withdrawn": ("bdg-secondary", "Withdrawn"), +} +KI_SEVERITIES = ("critical", "major", "minor", "nit") + +EB_REQUIRED = ("id", "engine", "versions", "topic", "title", "engine_does", "powderline_does", "you_see", "status") +EB_OPTIONAL = ("seen_in", "note") +KI_REQUIRED = ("id", "title", "status", "what_happens", "what_to_do") +KI_OPTIONAL = ("severity", "applies_to") + +# --- loading and validation ---------------------------------------------------------------------- + + +class CatalogError(Exception): + """The catalogue breaks one of its own rules.""" + + +def _id_key(entry_id: str) -> tuple[str, int]: + prefix, number = entry_id.split("-") + return prefix, int(number) + + +def _label(entry_id: str) -> str: + return entry_id.lower() + + +def _load(path: Path) -> list[dict]: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(data, list): + raise CatalogError(f"{path.name}: the file must be a YAML list of entries") + return data + + +def _check_entries(entries: list[dict], *, name: str, prefix: str, required: tuple, optional: tuple) -> list[str]: + problems: list[str] = [] + seen: set[str] = set() + for i, e in enumerate(entries): + where = f"{name} entry {i + 1}" + if not isinstance(e, dict): + problems.append(f"{where}: not a mapping") + continue + eid = e.get("id") + if isinstance(eid, str): + where = f"{name} {eid}" + if not isinstance(eid, str) or not re.fullmatch(rf"{prefix}-\d{{2,}}", eid): + problems.append(f"{where}: id must look like {prefix}-NN, got {eid!r}") + elif eid in seen: + problems.append(f"{where}: duplicate id") + else: + seen.add(eid) + for key in required: + if key not in e: + problems.append(f"{where}: missing field {key!r}") + for key in e: + if key not in required and key not in optional: + problems.append(f"{where}: unknown field {key!r}") + for key, value in e.items(): + if key in ("you_see", "seen_in"): + if not isinstance(value, list) or not all(isinstance(v, str) and v.strip() for v in value): + problems.append(f"{where}: {key} must be a list of non-empty strings") + elif not isinstance(value, str) or not value.strip(): + problems.append(f"{where}: {key} must be a non-empty string") + return problems + + +def validate(eb: list[dict], ki: list[dict]) -> list[str]: + """Every rule of the catalogue; returns the problems (empty when valid).""" + problems = _check_entries(eb, name="engine_behaviour.yaml", prefix="EB", required=EB_REQUIRED, + optional=EB_OPTIONAL) + problems += _check_entries(ki, name="known_issues.yaml", prefix="KI", required=KI_REQUIRED, + optional=KI_OPTIONAL) + if problems: + return problems + for e in eb: + if e["engine"] not in ENGINES: + problems.append(f"{e['id']}: engine {e['engine']!r} is not one of {sorted(ENGINES)}") + if e["topic"] not in TOPICS: + problems.append(f"{e['id']}: topic {e['topic']!r} is not one of {list(TOPICS)}") + if e["status"] not in EB_STATUSES: + problems.append(f"{e['id']}: status {e['status']!r} is not one of {list(EB_STATUSES)}") + for e in ki: + if e["status"] not in KI_STATUSES: + problems.append(f"{e['id']}: status {e['status']!r} is not one of {list(KI_STATUSES)}") + if "severity" in e and e["severity"] not in KI_SEVERITIES: + problems.append(f"{e['id']}: severity {e['severity']!r} is not one of {list(KI_SEVERITIES)}") + if "severity" not in e and e["status"] != "withdrawn": + problems.append(f"{e['id']}: missing field 'severity' (only a withdrawn entry may leave it out)") + # every id cited inside an entry's text must have an entry + known = {e["id"] for e in eb} | {e["id"] for e in ki} + for e in eb + ki: + for key, value in e.items(): + if key == "id": + continue + for text in value if isinstance(value, list) else [value]: + for m in ID_RE.finditer(text): + if m.group(0) not in known: + problems.append(f"{e['id']}: {key} cites {m.group(0)}, which has no entry") + return problems + + +# --- rendering ----------------------------------------------------------------------------------- + + +def _link_ids(text: str, self_id: str) -> str: + """Turn every cited ``EB-NN`` / ``KI-NN`` (other than the entry's own) into a ``{ref}`` link.""" + + def repl(m: re.Match) -> str: + cited = m.group(0) + return cited if cited == self_id else f"{{ref}}`{cited} <{_label(cited)}>`" + + return ID_RE.sub(repl, " ".join(text.split())) + + +def _code(item: str) -> str: + """A bare code (``easydiffraction_negative_width``) in backticks; anything else as written.""" + return f"`{item}`" if re.fullmatch(r"[a-z][a-z0-9_]*", item) else item + + +def render_eb_entry(e: dict) -> list[str]: + role, text = EB_STATUSES[e["status"]] + does = "What happens" if e["engine"] == "all" else f"{ENGINES[e['engine']]} does" + lines = [ + f"({_label(e['id'])})=", + f"#### {e['id']} · {' '.join(e['title'].split())}", + "", + f"{{{role}}}`{text}` · {' '.join(e['versions'].split())}", + "", + f"**{does}:** {_link_ids(e['engine_does'], e['id'])}", + "", + f"**PowderLine does:** {_link_ids(e['powderline_does'], e['id'])}", + "", + ] + if e["you_see"]: + lines += [f"**You see:** {', '.join(_code(s) for s in e['you_see'])}", ""] + if e.get("seen_in"): + lines += [f"**Seen in:** {', '.join(f'`{s}`' for s in e['seen_in'])}", ""] + if e.get("note"): + lines += [f"**Note:** {_link_ids(e['note'], e['id'])}", ""] + return lines + + +def render_eb(eb: list[dict], engine: str) -> str: + entries = [e for e in eb if e["engine"] == engine] + lines = [HEADER, ""] + for topic, heading in TOPICS.items(): + group = [e for e in entries if e["topic"] == topic] + if not group: + continue + lines += [f"### {heading}", ""] + for e in group: + lines += render_eb_entry(e) + return "\n".join(lines).rstrip("\n") + "\n" + + +def render_known_issues(ki: list[dict]) -> str: + lines = [HEADER, ""] + for status, (role, heading) in KI_STATUSES.items(): + group = sorted((e for e in ki if e["status"] == status), key=lambda e: _id_key(e["id"])) + if not group: + continue + lines += [f"## {heading}", ""] + for e in group: + meta = [f"{{{role}}}`{status}`"] + [e[k] for k in ("severity", "applies_to") if k in e] + lines += [ + f"({_label(e['id'])})=", + f"### {e['id']} · {' '.join(e['title'].split())}", + "", + " · ".join(meta), + "", + f"**What happens:** {_link_ids(e['what_happens'], e['id'])}", + "", + f"**What to do:** {_link_ids(e['what_to_do'], e['id'])}", + "", + ] + return "\n".join(lines).rstrip("\n") + "\n" + + +def render_ids(eb: list[dict], ki: list[dict]) -> str: + def ref(entry_id: str) -> str: + return f"{{ref}}`{entry_id} <{_label(entry_id)}>`" + + def cell(text: str) -> str: + return " ".join(text.split()).replace("|", "\\|") + + lines = [HEADER, "", "## Engine behaviours", "", "| ID | Engine | Title | Status |", "|---|---|---|---|"] + for e in sorted(eb, key=lambda e: _id_key(e["id"])): + lines.append(f"| {ref(e['id'])} | {ENGINES[e['engine']]} | {cell(e['title'])} | " + f"{EB_STATUSES[e['status']][1]} |") + lines += ["", "## Known issues", "", "| ID | Applies to | Title | Status |", "|---|---|---|---|"] + for e in sorted(ki, key=lambda e: _id_key(e["id"])): + lines.append(f"| {ref(e['id'])} | {cell(e.get('applies_to', '—'))} | {cell(e['title'])} | {e['status']} |") + return "\n".join(lines) + "\n" + + +def fragments(eb: list[dict], ki: list[dict]) -> dict[str, str]: + files = {f"eb_{engine}.md": render_eb(eb, engine) for engine in ENGINES} + files["ids.md"] = render_ids(eb, ki) + files["known_issues.md"] = render_known_issues(ki) + return files + + +# --- the id scan (--check) ----------------------------------------------------------------------- + + +def cited_ids(roots=SCAN_ROOTS) -> tuple[dict[str, str], int]: + """Every ``EB-NN`` / ``KI-NN`` in the text files under ``roots``: id -> first ``path:line``.""" + found: dict[str, str] = {} + n_files = 0 + for root in roots: + for path in sorted(root.rglob("*")): + if not path.is_file() or SCAN_SKIP_DIRS.intersection(path.parts): + continue + if any(path == skip or skip in path.parents for skip in SCAN_SKIP): + continue + try: + text = path.read_text(encoding="utf-8") + except (UnicodeDecodeError, OSError): + continue # binary or unreadable: not a place an id is written + n_files += 1 + for lineno, line in enumerate(text.splitlines(), 1): + for m in ID_RE.finditer(line): + shown = path.relative_to(ROOT) if ROOT in path.parents else path + found.setdefault(m.group(0), f"{shown.as_posix()}:{lineno}") + return found, n_files + + +# --- main ---------------------------------------------------------------------------------------- + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--check", action="store_true", + help="exit 1 if a committed fragment is stale or an id in src/ or docs/ has no entry") + ap.add_argument("--out-dir", type=Path, default=OUT_DIR) + args = ap.parse_args(argv) + + try: + eb, ki = _load(EB_FILE), _load(KI_FILE) + except (CatalogError, yaml.YAMLError) as exc: + print(f"catalog-check: cannot read the catalogue: {exc}", file=sys.stderr) + return 1 + problems = validate(eb, ki) + if problems: + print("catalog-check: the catalogue is invalid:\n " + "\n ".join(problems), file=sys.stderr) + return 1 + + files = fragments(eb, ki) + if not args.check: + args.out_dir.mkdir(parents=True, exist_ok=True) + for name, text in files.items(): + path = args.out_dir / name + with open(path, "w", encoding="utf-8", newline="\n") as fh: + fh.write(text) + print(f"wrote {path}") + return 0 + + status = 0 + stale = [args.out_dir / name for name, text in files.items() + if not (args.out_dir / name).exists() or (args.out_dir / name).read_text(encoding="utf-8") != text] + if stale: + print("stale (regenerate with scripts/gen_catalog.py): " + ", ".join(map(str, stale)), file=sys.stderr) + status = 1 + known = {e["id"] for e in eb} | {e["id"] for e in ki} + cited, n_files = cited_ids() + missing = sorted((i for i in cited if i not in known), key=_id_key) + if missing: + print("ids without a catalogue entry (add them to docs/catalog/*.yaml): " + + ", ".join(f"{i} (first at {cited[i]})" for i in missing), file=sys.stderr) + status = 1 + if status: + return status + try: + shown = args.out_dir.resolve().relative_to(Path.cwd().resolve()) + except ValueError: + shown = args.out_dir + print(f"catalog-check: {len(files)} files under {shown} match docs/catalog ({len(eb)} EB, {len(ki)} KI); " + f"the {len(cited)} ids cited in {n_files} files under src/ and docs/ all have entries") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/gen_example_figures.py b/scripts/gen_example_figures.py new file mode 100644 index 0000000..c79af97 --- /dev/null +++ b/scripts/gen_example_figures.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Draw the examples gallery's fit figures from the committed fit profiles. + +Every example with a committed ``fit_profile.txt`` gets one small SVG under +``docs/_generated/examples/``: + +* ``examples//output/fit_profile.txt`` -> ``.svg`` (the GSAS-II recipe); +* ``examples///output/fit_profile.txt`` -> ``_.svg``. + +Each figure shows the observed pattern (grey), the calculated pattern (blue) +and the difference curve (orange, offset below) over the fit window, with a +2θ axis. The colours are mid-tones and the background is transparent, so a +figure reads on the light and on the dark theme. + +The script is pure Python (no matplotlib, no engine), so it runs in the default +environment. Its output is byte-deterministic on every platform: the profiles +are reduced to a fixed number of columns (the minimum and maximum of each, so +peak heights survive), coordinates are rounded to 0.1 px, and files are +written as UTF-8 with ``\\n`` newlines:: + + python scripts/gen_example_figures.py # rewrite the figures + python scripts/gen_example_figures.py --check # exit 1 if a figure is stale, missing or extra + +Run it after a committed fit profile changes and commit the figures with the +change. The figures are drawn from the committed references only; the script +never runs a refinement. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EXAMPLES = ROOT / "examples" +OUT_DIR = ROOT / "docs" / "_generated" / "examples" + +HEADER = "" + +#: Engine subfolders of an example, in output order. The top-level recipe is GSAS-II's. +ENGINE_DIRS = ("easydiffraction", "topas") +ENGINE_LABELS = {"": "GSAS-II", "easydiffraction": "easydiffraction", "topas": "TOPAS"} + +# Canvas and plot area (px). +WIDTH, HEIGHT = 640, 300 +LEFT, RIGHT = 40.0, 624.0 +TOP, BOTTOM = 12.0, 250.0 +BIN_PX = 2.0 # one min/max pair per 2 px column + +# Mid-tone colours, legible on a light and on a dark background. +OBS = "#8c8c8c" +CALC = "#3d7fd9" +DIFF = "#e07b28" +AXIS = "#8c8c8c" + + +def profiles() -> list[tuple[str, str, str, Path]]: + """``(figure name, example, engine folder, fit profile)`` for every committed fit profile, in a fixed order.""" + found = [] + for example in sorted(p for p in EXAMPLES.iterdir() if p.is_dir() and p.name.startswith("example_")): + if example.name == "example_template": + continue + for engine in ("",) + ENGINE_DIRS: + path = (example / engine / "output" / "fit_profile.txt") if engine else (example / "output" / "fit_profile.txt") + if path.is_file(): + found.append((f"{example.name}_{engine}" if engine else example.name, example.name, engine, path)) + return found + + +def without_profile() -> list[str]: + """The example recipes (```` or ``/``) that have no committed fit profile.""" + missing = [] + for example in sorted(p for p in EXAMPLES.iterdir() if p.is_dir() and p.name.startswith("example_")): + if example.name == "example_template": + continue + for engine in ("",) + ENGINE_DIRS: + base = example / engine if engine else example + if (base / "input.json").is_file() and not (base / "output" / "fit_profile.txt").is_file(): + missing.append(f"{example.name}/{engine}" if engine else example.name) + return missing + + +def read_profile(path: Path) -> dict[str, list[float]]: + """The columns of a fit profile by their header names (tab-separated, one header line).""" + lines = path.read_text(encoding="utf-8").splitlines() + names = lines[0].split() + columns: dict[str, list[float]] = {n: [] for n in names} + for line in lines[1:]: + if not line.strip(): + continue + for name, field in zip(names, line.split()): + columns[name].append(float(field)) + for needed in ("two_theta", "y_obs", "y_calc"): + if needed not in columns: + raise SystemExit(f"{path}: no '{needed}' column in the header {names}") + if "y_diff" not in columns: + columns["y_diff"] = [o - c for o, c in zip(columns["y_obs"], columns["y_calc"])] + return columns + + +def fit_window(columns: dict[str, list[float]]) -> range: + """The rows of the fit window: GSAS-II writes the whole pattern with ``y_calc`` 0 outside it.""" + calc = columns["y_calc"] + inside = [i for i, v in enumerate(calc) if v != 0.0] + if not inside: + return range(len(calc)) + return range(inside[0], inside[-1] + 1) + + +def envelope(xs: list[float], ys: list[float], x0: float, x1: float, bins: int) -> list[tuple[float, float]]: + """Reduce a curve to the minimum and maximum of each x column, in x order, so peaks keep their height.""" + columns: list[list[tuple[float, float]]] = [[] for _ in range(bins)] + span = x1 - x0 + for x, y in zip(xs, ys): + k = int((x - x0) / span * bins) if span > 0 else 0 + columns[min(max(k, 0), bins - 1)].append((x, y)) + out: list[tuple[float, float]] = [] + for pts in columns: + if not pts: + continue + lo = min(pts, key=lambda p: p[1]) + hi = max(pts, key=lambda p: p[1]) + out.extend(sorted({lo, hi})) + return out + + +def fmt(v: float) -> str: + s = f"{v:.1f}" + return "0.0" if s == "-0.0" else s + + +def nice_step(span: float, max_ticks: int = 8) -> float: + for step in (0.1, 0.2, 0.25, 0.5, 1.0, 2.0, 2.5, 5.0, 10.0, 20.0, 25.0, 50.0, 100.0): + if span / step <= max_ticks: + return step + return 100.0 + + +def tick_label(v: float, step: float) -> str: + return f"{v:.0f}" if step >= 1 else f"{v:.1f}" if step * 10 == int(step * 10) else f"{v:.2f}" + + +def svg(example: str, engine: str, path: Path) -> str: + cols = read_profile(path) + rows = fit_window(cols) + tth = [cols["two_theta"][i] for i in rows] + obs = [cols["y_obs"][i] for i in rows] + calc = [cols["y_calc"][i] for i in rows] + diff = [cols["y_diff"][i] for i in rows] + x0, x1 = tth[0], tth[-1] + + # One intensity scale for all three curves; the difference curve sits below the patterns. + top_hi = max(max(obs), max(calc)) + top_lo = min(min(obs), min(calc)) + d_hi, d_lo = max(diff), min(diff) + gap = 0.04 * (top_hi - top_lo) + zero = top_lo - gap - d_hi # the difference curve's zero, in intensity units + y_hi, y_lo = top_hi, zero + d_lo + if y_hi == y_lo: + y_hi += 1.0 + + def px(x: float) -> float: + return LEFT + (x - x0) / (x1 - x0) * (RIGHT - LEFT) + + def py(y: float) -> float: + return BOTTOM - (y - y_lo) / (y_hi - y_lo) * (BOTTOM - TOP) + + bins = int((RIGHT - LEFT) / BIN_PX) + + def polyline(ys: list[float], colour: str, width: str, offset: float = 0.0) -> str: + pts = envelope(tth, ys, x0, x1, bins) + coords = " ".join(f"{fmt(px(x))},{fmt(py(y + offset))}" for x, y in pts) + return (f'') + + label = ENGINE_LABELS[engine] + source = path.relative_to(ROOT).as_posix() + + step = nice_step(x1 - x0) + first = step * int(x0 / step + (0 if x0 % step == 0 else 1)) + ticks = [] + t = first + while t <= x1 + 1e-9: + ticks.append(t) + t = round(t + step, 6) + + out = [ + HEADER, + f"", + f'', + f"{example} ({label}): observed (grey), calculated (blue) and difference (orange) " + f"over the fit window", + f'', + f'', + ] + for t in ticks: + x = fmt(px(t)) + out.append(f'') + out.append(f'') + for t in ticks: + out.append(f'{tick_label(t, step)}') + out.append(f'2θ (°)') + out.append(f'' + f"Intensity") + out.append("") + out.append(f'') + out.append(polyline(obs, OBS, "2")) + out.append(polyline(calc, CALC, "1")) + out.append(polyline(diff, DIFF, "1", offset=zero)) + out.append("") + return "\n".join(out) + "\n" + + +def figures() -> dict[str, str]: + return {f"{name}.svg": svg(example, engine, path) for name, example, engine, path in profiles()} + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--check", action="store_true", + help="exit 1 if a committed figure differs from the fit profiles, or is missing or extra") + ap.add_argument("--out-dir", type=Path, default=OUT_DIR) + args = ap.parse_args(argv) + wanted = figures() + skipped = without_profile() + try: + shown = args.out_dir.resolve().relative_to(Path.cwd().resolve()).as_posix() + except ValueError: + shown = str(args.out_dir) + + if not args.check: + args.out_dir.mkdir(parents=True, exist_ok=True) + for name, text in wanted.items(): + with open(args.out_dir / name, "w", encoding="utf-8", newline="\n") as fh: + fh.write(text) + print(f"wrote {shown}/{name}") + for extra in sorted(p.name for p in args.out_dir.glob("*.svg") if p.name not in wanted): + print(f"note: {shown}/{extra} has no fit profile behind it; delete it", file=sys.stderr) + if skipped: + print("no committed fit profile (no figure): " + ", ".join(skipped)) + return 0 + + stale, missing = [], [] + for name, text in wanted.items(): + path = args.out_dir / name + if not path.is_file(): + missing.append(name) + elif path.read_bytes() != text.encode("utf-8"): + stale.append(name) + extra = sorted(p.name for p in args.out_dir.glob("*.svg") if p.name not in wanted) if args.out_dir.is_dir() else [] + if stale or missing or extra: + parts = [f"{label}: " + ", ".join(names) + for label, names in (("stale", stale), ("missing", missing), ("extra", extra)) if names] + print(f"figures-check: under {shown}, " + "; ".join(parts) + + " (regenerate with scripts/gen_example_figures.py; delete an extra figure)", file=sys.stderr) + return 1 + print(f"figures-check: {len(wanted)} figures under {shown} match the committed fit profiles " + f"({len(skipped)} example recipes have none)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 4d4e69e758b3dc931fb6d17c21d27f765af51871 Mon Sep 17 00:00:00 2001 From: acorrao <95718914+AdamCorrao@users.noreply.github.com> Date: Thu, 5 Nov 2026 16:44:40 -0500 Subject: [PATCH 03/18] docs: rebuild the documentation around readers and tasks New structure: Get started (tutorials), Concepts, How-to guides, Examples (gallery with labelled teaching examples), Engines (per-engine reference with every EB entry anchored), Reference, Development and About. Pages are MyST Markdown, describe the current release only, and keep one home per fact. Material that was internal process or history (the regression-tolerance study, the cross-platform analysis, the unified-schema archive, known-issue disposition rounds) leaves the public docs. README and landing page lead with what PowderLine is and why; CONTRIBUTING and AGENTS point at the new pages; CHANGELOG links follow the moves. The two tests that read docs pages read the new ones. Assisted-by: claude-code:claude-opus-5-5 --- AGENTS.md | 371 +++---- CHANGELOG.md | 26 +- CONTRIBUTING.md | 83 +- README.md | 353 +++---- docs/DEVELOPMENT.md | 719 ------------- docs/SCHEMA_HISTORY.md | 812 --------------- docs/TROUBLESHOOTING.md | 943 ------------------ docs/{ => about}/changelog.md | 2 +- docs/{ => about}/citing.md | 6 +- docs/about/index.md | 18 + docs/api.rst | 209 ---- .../comparing-engines.md} | 32 +- docs/concepts/how-powderline-works.md | 182 ++++ docs/concepts/index.md | 23 + docs/concepts/recipes.md | 145 +++ docs/concepts/refinement-practice.md | 125 +++ docs/concepts/results.md | 148 +++ docs/concepts/validation.md | 163 +++ docs/cross-platform-guide.md | 593 ----------- docs/development/adding-an-engine.md | 203 ++++ docs/development/architecture.md | 342 +++++++ docs/development/contributing.md | 246 +++++ docs/development/index.md | 32 + docs/development/internals-api.md | 122 +++ docs/development/known-issues.md | 29 + docs/development/migrating-from-0.1.md | 77 ++ docs/development/policies.md | 184 ++++ docs/engine-behavior.md | 674 ------------- docs/engine-versions.md | 80 -- docs/engines/easydiffraction.md | 243 +++++ docs/engines/gsasii.md | 254 +++++ docs/engines/index.md | 178 ++++ docs/engines/topas.md | 277 +++++ docs/examples/index.md | 132 +++ docs/fields.md | 109 -- docs/getting-started.rst | 290 ------ docs/getting-started/first-refinement.md | 134 +++ docs/getting-started/index.md | 20 + docs/getting-started/install.md | 77 ++ docs/getting-started/your-own-data.md | 172 ++++ docs/how-to/beamline-operation.md | 201 ++++ docs/how-to/calibrate-instrument.md | 205 ++++ docs/how-to/disorder.md | 119 +++ docs/how-to/index.md | 31 + docs/how-to/python-pipelines.md | 262 +++++ docs/how-to/refine-structure.md | 284 ++++++ docs/how-to/run-topas.md | 149 +++ docs/how-to/sequential-refinement.md | 144 +++ docs/how-to/simulate.md | 146 +++ docs/how-to/single-peak-fitting.md | 241 +++++ docs/how-to/troubleshooting.md | 467 +++++++++ docs/index.md | 103 ++ docs/index.rst | 53 - docs/input-guidance.md | 529 ---------- docs/integration-guide.md | 328 ------ docs/known_issues.md | 832 --------------- docs/migration-0.1-to-0.2.md | 420 -------- docs/policies.md | 154 --- docs/quickstart.md | 99 -- docs/reference/api.md | 95 ++ docs/reference/cli.md | 132 +++ docs/reference/fields.md | 53 + docs/reference/glossary.md | 161 +++ docs/reference/ids.md | 4 + docs/reference/index.md | 30 + docs/reference/messages.md | 125 +++ docs/reference/outputs.md | 297 ++++++ docs/reference/recipe-format.md | 273 +++++ docs/reference/scope.md | 63 ++ docs/reference/versions.md | 201 ++++ docs/regression-tolerance.md | 183 ---- docs/schemas.md | 229 ----- docs/support-matrix.md | 101 -- examples/CONTRIBUTING_EXAMPLES.md | 31 +- examples/example_DRX_33/DESCRIPTION.md | 74 +- .../example_DRX_33_anisoADP/DESCRIPTION.md | 18 +- .../example_DRX_33_atomrefine/DESCRIPTION.md | 29 +- .../example_DRX_33_simulation/DESCRIPTION.md | 6 +- .../example_DRX_33_strainonly/DESCRIPTION.md | 16 +- examples/example_LaB6/DESCRIPTION.md | 60 +- examples/example_LaB6_ionic/DESCRIPTION.md | 11 +- .../example_LaB6_simulation/DESCRIPTION.md | 14 +- .../example_LaB6_singlepeakfit/DESCRIPTION.md | 27 +- .../DESCRIPTION.md | 4 +- examples/example_LaB6_textured/DESCRIPTION.md | 2 +- examples/example_template/DESCRIPTION.md | 6 +- tests/test_examples_validate.py | 12 +- tests/test_warning_codes.py | 4 +- 88 files changed, 7795 insertions(+), 8061 deletions(-) delete mode 100644 docs/DEVELOPMENT.md delete mode 100644 docs/SCHEMA_HISTORY.md delete mode 100644 docs/TROUBLESHOOTING.md rename docs/{ => about}/changelog.md (87%) rename docs/{ => about}/citing.md (95%) create mode 100644 docs/about/index.md delete mode 100644 docs/api.rst rename docs/{cross-engine-comparison.md => concepts/comparing-engines.md} (89%) create mode 100644 docs/concepts/how-powderline-works.md create mode 100644 docs/concepts/index.md create mode 100644 docs/concepts/recipes.md create mode 100644 docs/concepts/refinement-practice.md create mode 100644 docs/concepts/results.md create mode 100644 docs/concepts/validation.md delete mode 100644 docs/cross-platform-guide.md create mode 100644 docs/development/adding-an-engine.md create mode 100644 docs/development/architecture.md create mode 100644 docs/development/contributing.md create mode 100644 docs/development/index.md create mode 100644 docs/development/internals-api.md create mode 100644 docs/development/known-issues.md create mode 100644 docs/development/migrating-from-0.1.md create mode 100644 docs/development/policies.md delete mode 100644 docs/engine-behavior.md delete mode 100644 docs/engine-versions.md create mode 100644 docs/engines/easydiffraction.md create mode 100644 docs/engines/gsasii.md create mode 100644 docs/engines/index.md create mode 100644 docs/engines/topas.md create mode 100644 docs/examples/index.md delete mode 100644 docs/fields.md delete mode 100644 docs/getting-started.rst create mode 100644 docs/getting-started/first-refinement.md create mode 100644 docs/getting-started/index.md create mode 100644 docs/getting-started/install.md create mode 100644 docs/getting-started/your-own-data.md create mode 100644 docs/how-to/beamline-operation.md create mode 100644 docs/how-to/calibrate-instrument.md create mode 100644 docs/how-to/disorder.md create mode 100644 docs/how-to/index.md create mode 100644 docs/how-to/python-pipelines.md create mode 100644 docs/how-to/refine-structure.md create mode 100644 docs/how-to/run-topas.md create mode 100644 docs/how-to/sequential-refinement.md create mode 100644 docs/how-to/simulate.md create mode 100644 docs/how-to/single-peak-fitting.md create mode 100644 docs/how-to/troubleshooting.md create mode 100644 docs/index.md delete mode 100644 docs/index.rst delete mode 100644 docs/input-guidance.md delete mode 100644 docs/integration-guide.md delete mode 100644 docs/known_issues.md delete mode 100644 docs/migration-0.1-to-0.2.md delete mode 100644 docs/policies.md delete mode 100644 docs/quickstart.md create mode 100644 docs/reference/api.md create mode 100644 docs/reference/cli.md create mode 100644 docs/reference/fields.md create mode 100644 docs/reference/glossary.md create mode 100644 docs/reference/ids.md create mode 100644 docs/reference/index.md create mode 100644 docs/reference/messages.md create mode 100644 docs/reference/outputs.md create mode 100644 docs/reference/recipe-format.md create mode 100644 docs/reference/scope.md create mode 100644 docs/reference/versions.md delete mode 100644 docs/regression-tolerance.md delete mode 100644 docs/schemas.md delete mode 100644 docs/support-matrix.md diff --git a/AGENTS.md b/AGENTS.md index 26cd373..633f848 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,243 +1,148 @@ # AGENTS.md — PowderLine -Guidance for coding agents (and a quick orientation for human contributors). -Tool-specific files (`CLAUDE.md`) only point here. - -Automated powder X-ray diffraction refinement driven from a JSON "recipe". -PowderLine is the application layer over interchangeable refinement engines: -a recipe names **one engine and one workflow** in its `schema_name` -(`gsasii.rietveld`, `gsasii.spf`, `topas.rietveld`, `topas.spf`, -`easydiffraction.rietveld`), is written in that engine's own conventions and -units, and `powderline.run()` / the `powderline` CLI hand it to that engine's -**gateway** and return standardized tables with uniform fit statistics. -Nothing is translated between engines. Example-driven: real refinement cases -drive schema evolution and are the regression references. - -**Authoritative sources for a change:** `docs/policies.md` (versioning, -deliberate changes, frozen code, engine pins, review, "ask, don't -assume"), `docs/SCHEMA_HISTORY.md` (schema facts), `docs/engine-behavior.md` -(what the engines do and PowderLine's policy), `docs/known_issues.md` -(`KI-NN`). Maintainers who work with the private design records (the devkit) -also follow the devkit's own agent guidance. **If a decision is not written down, -stop and ask the maintainer; do not infer it from the code or from another -engine.** - -## Architecture (`src/powderline/`) +A path map for coding agents (and a quick orientation for human +contributors). Tool-specific files (`CLAUDE.md`) only point here. This file +links the page that owns each topic instead of restating it. + +PowderLine runs powder diffraction refinements from a **recipe** (structured +data: a dict, serialised as JSON). A recipe names one engine and one workflow +in `schema_name` (`gsasii.rietveld`, `gsasii.spf`, `topas.rietveld`, +`topas.spf`, `easydiffraction.rietveld`) and is written in that engine's own +terms and units. `powderline.run()` and the `powderline` CLI hand it to the +engine's **gateway** and return standardized tables with uniform fit +statistics. Nothing is translated between engines. + +## Authoritative sources + +- `docs/development/policies.md` — versioning, deliberate changes, frozen + code, engine pins, review, "ask, don't assume". +- `docs/reference/versions.md` — schema versions, history, support matrix. +- `docs/engines/*.md` + `docs/catalog/engine_behaviour.yaml` — what each + engine does and PowderLine's policy (`EB-NN`). +- `docs/development/known-issues.md` + `docs/catalog/known_issues.yaml` — + known, deliberately deferred behaviours (`KI-NN`). Check before "fixing" a + quirk. +- The code under `src/` and `pixi.toml`: where a page and the code disagree, + the code wins; report the discrepancy. + +**If a decision is not written down, stop and ask the maintainer.** Do not +infer it from the code or from another engine. Maintainers working with the +private devkit also follow its own agent guidance. + +## Code map (`src/powderline/`) Core (engine-free; `import powderline` imports no engine): -- `__init__.py` — the public API, `__all__`: `run`, `validate`, `ValidationResult`, - `load_recipe_asset`, `support_matrix`, `PowderLineError`, - `GatewayNotInstalledError`, `EngineNotAvailableError`, `EngineVersionError`, - `EngineExecutionError`, `ExecutionBackendError`. -- `api.py` — `validate(recipe)` → `ValidationResult(recipe=, - warnings=[...])` through the gateway's engine-free schema layer; - `run(recipe, output_dir, *, verbose, validate_only, **engine_options)` - dispatches on `schema_name` (`.` → `registry.get(engine)`); - engine options are the gateway's explicit keywords (gsasii `execution_mode`; - topas `topas_dir`, `topas_version`, `topas_timeout`); `allow_unsupported_engine_version` is - refused (test-only, on the gateway's own `run`). -- `kicker.py` — the one CLI: `powderline [--output DIR] - [--validate-only] [--verbose] [--use-server|--no-server] [--server-timeout SECONDS] - [--topas-dir DIR] [--topas-version V] [--topas-timeout SECONDS]` and `powderline --doctor` (`doctor.py`: gateways, - engine versions vs pins, server state; non-zero on a stale install, KI-13); - `pixi run kicker` / `pixi run doctor` run the same `main`. Refuses - templates; an engine flag for another engine's recipe exits 1. -- `registry.py` — gateways from the `powderline.gateways` entry-point group - (lazy `ep.load`), the frozen `Gateway` dataclass (`name`, `capabilities`, - `validate`, `run`), `register_gateway` escape hatch, `check_engine_version`. - Entry points only; no in-code fallback (KI-13). -- `schema_core.py` — core schema **0.1.0**: `CoreRecipe` frame - (`schema_name`, `core_schema_version`, `engine_schema_version`, `metadata` - ≤ 1 MiB, `payload`), `RefinableParameter` `[value, flag]` / - `BoundedRefinableParameter` `[value, flag, min, max]`, `XRDData`, `FitRange`, - `ChebyshevBackground`, the phase block `Phase[P]` (`space_group` canonical - gemmi name, `unit_cell`, `atoms`; every symmetry rule: cell check, special - positions, tie groups and flags, derived values held exactly), `CoreName`, - `collect_warnings`, `parameters_requested`, `fit_limits_on_data`, and the - engine schema building blocks (`EngineRecipe`, `recipe_validator`, - `refined_flags`, `custom_error`/`raise_problems`, `omit_absent`, - `absent_not_null`, `unique_phase_names`, `fit_window`, `fit_range_problems`, - `outside_window_problems`, `refined_flag_problems`, `unit_schema`) every - gateway schema is composed from. Every - model `extra='forbid'`; units in `json_schema_extra={"unit": ...}`. -- `symmetry.py` (gemmi), `compat.py` (declared schema compatibility, - `support_matrix()`), `fitstats.py` (uniform Rwp/Rexp/GoF/χ²_red), `reports.py` - (standardized report writers), `recipes.py`, `exceptions.py` (also - `WARNING_CODES`: every warning code with owner and level, asserted against - `src/` and the engine-behaviour table), - `config_loader.py` (simulation defaults), `_status.py`. -- `results.py` — the run-result envelope (`ENVELOPE_VERSION`, `REASONS`, the - failed/generated envelopes, `raise_if_failed`); `roundtrip.py` — the output - recipe (`output_recipe`, `output_recipe_valid`, `parameter_dispositions`; - tie propagation, KI-04); `doctor.py` — the `--doctor` report (KI-13). - -Gateways (`src/powderline/gateways//`), each a **schema layer** -(`schema.py`: the engine schema 0.1.0 on core 0.1.0, `DECLARATIONS`, -`SCHEMA_NAMES`, `validate_recipe`; imports no engine) plus a **runtime -layer** (`gateway.py` factory `get_gateway()` with `capabilities()`, -`validate()`, `run()`, `ENGINE_VERSION_SPEC`, `ENV`/`INSTALL_COMMAND`; engine -imported lazily from `run()`): -- `gsasii/` (GSAS-II `==5.7.9`, env `gsasii`): the **frozen** setup/execution - code `helpers.py`, `project.py`, `setters.py`, `executors.py`, - `extractors.py`, `constraints.py` (moved verbatim from `kicker.py` at the - verified baseline `daace24`; code-hashed); non-frozen `routing.py`, `fit_report.py` - (uniform statistics, `engine_details`, convergence, divergence = failure), - `sites.py` (GSAS-II's reading of each site vs core's), `report_rows.py`, - `paths.py` (GSAS-II parameter names → recipe paths for the output recipe), - `console.py` (the engine's console tail into a failed envelope, KI-05), - `scattering.py` (GSAS-II's ion table, `La3+` → `La+3`); - `server.py` / `client.py` (persistent FastAPI server + `GSASClient`, the - `auto`/`server` execution modes; per-user state dir, token-signed requests - and responses, `POWDERLINE_NO_SERVER=1` disables it — v0.1.2, KI-14; output files travel - back inside the response, no shared filesystem — KI-12); `simulation_builder.py` - (a crystal-structure dict → a `gsasii.rietveld` simulation recipe; the Materials Project - fetch/export tool that used it left PowderLine in 0.2.0; RandomXRD carries its own). -- `topas/` (TOPAS 6; recipes state `engine_version: "6"`, runs need the - installed version declared): `native_writer.py` (model → INP + `.xye`; - limits on data points; `= p/q;` constants), `native_run.py` (run `tc.exe`, - strict file reads, the background-only second run, result), `peak_list.py`, - `runner.py`, `scattering.py` (TOPAS 6's ion table, `La3+` → `occ La+3`). - No Python engine: generate-only - without `tc.exe`. -- `easydiffraction/` (`==0.21.1`, env `easydiffraction`, Python ≥ 3.12): - `native_builder.py`, `native_run.py`, `space_group_table.py`, - `space_group_support.py` (per-calculator verified settings), `scattering.py` - (CrysPy's ion table). - -Docs: `docs/schemas.md`, `docs/fields.md` (generated tables), `docs/input-guidance.md`, -`docs/engine-behavior.md`, `docs/policies.md`, `docs/migration-0.1-to-0.2.md`, -`docs/support-matrix.md`, `docs/cross-engine-comparison.md`, `docs/DEVELOPMENT.md` -("Adding a refinement engine"). Scripts: `scripts/gen_schema_field_tables.py` -(field tables + support matrix → `docs/_generated/`), `scripts/cross_engine_comparison.py`, -`scripts/convert_recipe_026.py` (0.26.0 → `gsasii.*` migration tool; -fixtures `tests/data/convert_026/`), `scripts/update_code_hash.py`. -## Key commands (pixi) - -Environments: `default` = core only (validate anything, run nothing); -`gsasii`, `easydiffraction`, `all` (both engines), `dev` (all + docs/test -tooling). `pixi install -e `. -- `pixi run kicker --validate-only` — any env; engine-free. -- `pixi run -e gsasii kicker ` — run a `gsasii.*` recipe - (`--use-server`/`--no-server`, `--output DIR`, `--verbose`); - `pixi run -e easydiffraction kicker ` for `easydiffraction.*`; - `pixi run kicker --topas-version 6 [--topas-dir DIR] - [--topas-timeout SECONDS]` for `topas.*` (runs `tc.exe` on Windows; when no - `tc.exe` is found the run is generate-only: the INP and `.xye` are written, - the result has status `generated` and the CLI exits 0). - After `pixi reinstall powderline` the installed command is `powderline …`. -- `pixi run test` — the engine-free suites (`core or topas_gen`, any env); - `pixi run -e gsasii test-gsasii`, `pixi run -e easydiffraction - test-easydiffraction` per engine; `pixi run test-topas` — `topas_gen` + - `topas_exec` (the exec tests run `tc.exe` only when `POWDERLINE_TOPAS_DIR` - and `POWDERLINE_TOPAS_VERSION` are set, on the TOPAS Windows machine; else - they skip); `pixi run test-all` — everything (`dev`). -- `pixi run -e gsasii gsas-server start|status|stop|restart|logs` — the - persistent GSAS-II server (task in the `gsasii` feature; without `-e` pixi - reports it ambiguous). -- `pixi run docs` / `pixi run docs-clean` — Sphinx HTML with `-W` (zero warnings). -- `PYTHONPATH=src python scripts/gen_schema_field_tables.py [--check]` — after - editing a schema module or a gateway's declarations (regenerates - `docs/_generated/fields_*.md`, `support_matrix.md`); `pixi run fields-check` - runs the `--check` form (fails when the committed fragments are stale). -- `pixi reinstall powderline --all` — after editing `[project.entry-points]` - or `[project.scripts]` in `pyproject.toml`: entry-point metadata is written - at install time, per env; the registry finds nothing until the editable - package is re-installed (KI-13). -- `pixi run update-code-hash` — regenerate `src/powderline/_code_hash.json` - after a **signed** edit to a frozen gsasii module (guarded by `tests/test_code_hash.py`). - -## Frozen GSAS-II code (the rule stays) +- `__init__.py` — public API (`__all__`): `run`, `validate`, `ValidationResult`, `load_recipe_asset`, `support_matrix`, the exceptions. +- `api.py` — `validate()` / `run()`: `schema_name` → `registry.get()`; clears fixed outputs, writes `failure.json`; refuses `allow_unsupported_engine_version`. +- `kicker.py` — the CLI (`powderline`); refuses templates; an engine flag for another engine's recipe exits 1. +- `doctor.py` — `powderline --doctor` (KI-13: stale install). +- `registry.py` — entry-point group `powderline.gateways`, `Gateway` dataclass, `register_gateway`, `check_engine_version`, `validate_only_summary`. No in-code fallback. +- `schema_core.py` — core schema 0.1.0: `CoreRecipe`, `RefinableParameter` / `BoundedRefinableParameter`, `XRDData`, `FitRange`, `ChebyshevBackground`, `Phase[P]`, `EngineRecipe`, `recipe_validator`, the building blocks. +- `symmetry.py` — gemmi: canonical space groups, cell ties, special positions. +- `compat.py` — declared schema compatibility, `schema_declarations`, `support_matrix`. +- `fitstats.py` — uniform Rwp / Rexp / GoF / χ²_red. +- `reports.py` — report writers, `FIXED_OUTPUT_NAMES` / `FIXED_OUTPUT_PATTERNS`, `clear_fixed_outputs`. +- `results.py` — result envelope, `REASONS`, `raise_if_failed`. +- `roundtrip.py` — output recipe and parameter dispositions. +- `exceptions.py` — exceptions and `WARNING_CODES`. +- `recipes.py`, `config_loader.py` (`.powderline_config.yaml`), `_status.py`, `_code_hash.json`. + +Gateways (`src/powderline/gateways//`): schema layer `schema.py` +(`DECLARATIONS`, `SCHEMA_NAMES`, `validate_recipe`; imports no engine) + +runtime layer `gateway.py` (`get_gateway()`, `ENGINE_VERSION_SPEC`, `ENV` / +`INSTALL_COMMAND`; engine imported lazily from `run()`). + +- `gsasii/` (GSAS-II `==5.7.9`, env `gsasii`) — frozen: `helpers`, `project`, `setters`, `executors`, `extractors`, `constraints`; around them: `routing`, `fit_report`, `sites`, `paths`, `report_rows`, `console`, `scattering`, `server` / `client` (persistent server, `POWDERLINE_NO_SERVER=1` disables it), `simulation_builder`. +- `topas/` (TOPAS 6, no Python engine; generate-only without `tc.exe`) — `native_writer`, `native_run`, `peak_list`, `runner`, `scattering`. +- `easydiffraction/` (`==0.21.1`, env `easydiffraction`, Python ≥ 3.12) — `native_builder`, `native_run`, `space_group_table`, `space_group_support`, `scattering`. + +Docs (`docs/`, MyST): `getting-started/`, `concepts/`, `how-to/`, +`examples/`, `engines/`, `reference/`, `development/`, `about/`; +`_generated/` (script output, never hand-edited); `catalog/` (EB/KI data). +`docs/reference/messages.md` holds the warning-codes table that +`tests/test_warning_codes.py` reads. + +Scripts (`scripts/`): `gen_schema_field_tables.py`, `gen_catalog.py`, +`gen_example_figures.py`, `cross_engine_comparison.py`, +`convert_recipe_026.py` (0.26.0 → `gsasii.*`), `check_engine_version_candidate.py`, +`update_code_hash.py`, `verify_easydiffraction_settings.py`, +`generate_easydiffraction_space_groups.py`, `hooks/pre-commit`. + +## Key commands + +Environments: `default` (core only: validates everything, runs nothing), +`gsasii`, `easydiffraction`, `all`, `dev` (all + docs and test tooling). +Install with `pixi install --frozen -e `. + +- `pixi run powderline --validate-only` — any env. +- `pixi run -e gsasii powderline [--use-server|--no-server] [--server-timeout S] [--output DIR] [--verbose]`. +- `pixi run -e easydiffraction powderline `. +- `pixi run powderline --topas-version 6 [--topas-dir DIR] [--topas-timeout S]` — without `tc.exe`: status `generated`, exit 0. +- `pixi run powderline --doctor`. The tasks `kicker` and `doctor` run the same `main`. +- `pixi run test` (engine-free: `core or topas_gen`); `pixi run -e gsasii test-gsasii`; `pixi run -e easydiffraction test-easydiffraction`; `pixi run test-topas` (runs `tc.exe` only with `POWDERLINE_TOPAS_DIR` and `POWDERLINE_TOPAS_VERSION` set); `pixi run test-all` (`dev`). +- `pixi run -e gsasii gsas-server start|status|stop|restart|logs`. +- `pixi run docs` / `pixi run docs-clean` — Sphinx, `-W`, zero warnings. +- `pixi run fields-check`, `pixi run catalog-check`, `pixi run figures-check` — generated docs are current; regenerate with the matching script without `--check`. +- `pixi run update-code-hash` — after any edit to a frozen gsasii module. +- `pixi reinstall powderline --all` — after editing `[project.entry-points]` or `[project.scripts]` (KI-13). + +Detail: `docs/development/contributing.md`. + +## Frozen GSAS-II code `src/powderline/gateways/gsasii/{helpers,project,setters,executors,extractors,constraints}.py` -are **move-only**: they do GSAS-II's project-dictionary work exactly as the -pre-refactor code did. Never change what they do to the GSAS-II project dict. -Additive or input-side edits — and output-side control flow that leaves the -project-dictionary work untouched (the failure propagation, the checks -before any result table is written) — go **only** through the maintainers' -signed-off record of frozen-code edits, each entry with its proof, then -`pixi run update-code-hash`. An agent never makes such an edit on its own: -propose it to the maintainer. New behaviour goes in -the non-frozen modules (`fit_report.py`, `sites.py`, `routing.py`, -`gateway.py`). Restart the GSAS-II server after any edit it runs (the client -compares the server's source-tree hash with its own and refuses a stale one). - -## Engines / environment notes - -- `GSAS-II` is built as a **conda package via `pixi-build`** from the - `tacaswell/GSAS-II@enh/pixi_pkg` fork (pinned to a commit in `pixi.toml`, - feature `gsasii`); it resolves to **v5.7.9**, which the gateway's - `ENGINE_VERSION_SPEC` checks at run time. easydiffraction is pinned - `==0.21.1`; gemmi `==0.7.5` (the canonical space-group names depend on it). - Moving a pin is a deliberate change with a compatibility run - (`docs/policies.md`): GSAS-II (or scipy/BLAS) changes shift refined values - at the last digit, so `refined_parameters.csv` and the unit-cell reports are - compared within tolerance (`docs/regression-tolerance.md`; - `tests/subprocess_utils.py` `refined_params_match` / `unit_cell_match`) and - peak lists by row count. TOPAS and easydiffraction goldens live under - `tests/data/{topas,easydiffraction}/native/golden/` (TOPAS's by version, - `golden/6/`); TOPAS runs only in the `topas_exec` tests on the TOPAS machine - (goldens from the TOPAS machine). -- The schema layer of every gateway, `powderline.api` and `powderline.kicker` - import no engine — enforced by subprocess tests that block the engine - imports. Keep it that way: engine imports live inside `run()`-time code. -- Platforms: `linux-64`, `win-64`, `osx-arm64`. **All functionality must work - on Linux and Windows** (sole exception: TOPAS *execution* needs `tc.exe` on - win-64). Avoid Unix-only shell syntax in pixi tasks — tasks set `PYTHONPATH` - through the task-level `env` dict; see `docs/cross-platform-guide.md`. -- **Testing caveat**: some `test_esd_extraction` / `test_example_*_regression` - cases require a working GSAS-II refinement; they can fail for environment - reasons unrelated to a given change. Compare against the baseline rather - than assuming a failure is new. +do GSAS-II's project-dictionary work. Rules (`docs/development/policies.md#frozen-gsas-ii-code`): + +- A behaviour-changing edit needs the maintainers' explicit consent. An agent + never makes one on its own: propose it, with its reason and proof. +- Comment and docstring edits need no consent. Don't leave outdated comments. +- After **any** edit, `pixi run update-code-hash` (the hash covers raw bytes; + `tests/test_code_hash.py` guards it). +- New behaviour goes into the non-frozen modules (`fit_report`, `sites`, + `routing`, `paths`, `gateway`). +- Restart the GSAS-II server after any edit it runs (a stale server is + detected and refused). + +## Environment notes + +- GSAS-II is a conda package built by `pixi-build` from a fork pinned to a + commit in `pixi.toml`; it resolves to v5.7.9. Use `--frozen` so it is not + re-locked. +- Pins: GSAS-II `==5.7.9`, easydiffraction `==0.21.1`, gemmi `==0.7.5`. Moving + a pin is a deliberate change with a compatibility run (policies page). +- Regression references: `examples/**/output/` (gsasii, easydiffraction), + compared within tolerance (`tests/subprocess_utils.py` + `refined_params_match` / `unit_cell_match`); goldens under + `tests/data/{topas,easydiffraction}/native/golden/` (TOPAS by version). +- The schema layers, `powderline.api` and `powderline.kicker` import no engine; + subprocess tests enforce it. +- Platforms: `linux-64`, `win-64`, `osx-arm64`. Everything works on Linux and + Windows except TOPAS execution (`tc.exe`, Windows). pixi tasks set + `PYTHONPATH` through the task-level `env` dict; no Unix-only shell syntax. +- Some GSAS-II regression and ESD tests can fail for environment reasons: + compare against a baseline run on `main`. ## Conventions -- Parameters are lists: `[value, refine_flag]` in `gsasii.*`, - `[value, refine_flag, min, max]` in `topas.*` and `easydiffraction.*`; the - flag is a JSON boolean; `null` bounds mean open. Blocks that may be left out - are left out, never `null`; unknown keys are errors. -- **Validation never changes a recipe beyond its stated tolerance, and never - silently**: anything an engine would silently alter is a validation error - naming the value to write; a value stated within tolerance of its exact - symmetry-derived form is *held* in that form and reported - (`symmetry_value_derived`, level `info`); documented defaults (gsasii - size/strain 10 µm / 0, background none, 5 cycles) and informative findings - are structured warnings `{code, message, field_path[, level]}` (`level` is - `"warning"` when absent, or `"info"`). -- Space groups are gemmi's canonical names (`"P m -3 m"`, `"F d -3 m:2"`); - every member of a symmetry tie group is stated, with the same flag. -- Results: uniform `rwp`/`r_exp`/`gof`/`chi2_red` from core `fitstats` over the - window's weighted points; the engine's own values and - `parameters_requested`/`parameters_varied` under `engine_details`; every - result carries `status` (`ok` / `generated`), `converged`, `output_recipe` + - `output_recipe_valid` + `parameter_dispositions` (KI-04: the resolved - recipe with refined values written back, built by `powderline.roundtrip`) and - `envelope_version` (no `success` key: `status` is the outcome); **every failure - raises**: `EngineExecutionError` (the engine ran and failed) or - `ExecutionBackendError` (it did not run), with `.reason` from - `powderline.results.REASONS` and `.result` the failed envelope (the - GSAS-II server speaks the same envelope). -- Output uses fixed filenames (`refined_parameters.csv`, - `_unit_cell_report.csv`, `_peak_list_report.csv`, - `fit_profile.txt`, `output_recipe.json`, `parameter_dispositions.csv`; - gsasii `dummy.gpx`/`dummy.lst`; topas `.inp` etc.). - `unit_cell_data`/`peak_list_data` from `run()` are dicts keyed by phase name. - `powderline.run()` removes PowderLine's own fixed-name files from the output - directory before every run (`reports.clear_fixed_outputs`) — only once the - recipe has validated and the engine is importable, so a typo or a wrong - environment costs nothing — and, when a run fails, writes the failed envelope - to `failure.json` beside the engine's own files; no result table is written - for a failed run on any engine. -- Known, deliberately-deferred behaviours are registered with stable `KI-NN` - ids in `docs/known_issues.md` — check it before "fixing" a quirk; engine - quirks and PowderLine's policy are on `docs/engine-behavior.md`. -- Every intentional behaviour change is a deliberate change (recorded, - reviewed, in the CHANGELOG); regression references under - `examples/**/output/` **are** tracked and move only with an equivalence - record — do not gitignore or regenerate them casually. +- Parameter shapes, bounds, units, unknown keys, blocks left out (never + `null`) — `docs/reference/recipe-format.md`. +- Validation never changes a recipe silently; space groups are gemmi's + canonical names; tie groups are stated in full — `docs/concepts/validation.md`. +- Structured warnings `{code, message, field_path[, level]}`; every code in + `exceptions.WARNING_CODES` and `docs/reference/messages.md`. +- Result keys, report files, `failure.json`, failure reasons — + `docs/reference/outputs.md`; uniform statistics and the output recipe — + `docs/concepts/results.md`. +- Every failure raises `EngineExecutionError` or `ExecutionBackendError`; a + gateway never returns a failed result — `docs/development/adding-an-engine.md`. +- Every intentional behaviour change is deliberate; references move only with + an equivalence check — `docs/development/policies.md`. +- Docs follow the writing rules in + `docs/development/contributing.md#writing-documentation`. - Don't commit secrets: `.powderline_config.yaml` is gitignored. -- `_dev/` is a reserved, gitignored path (the maintainers' private design - records). Never `git add` it or anything under it. +- The private devkit is reserved under `_dev/` (gitignored). Never commit it + or anything under it. - Never commit TOPAS's own files (console text, `tc.log`, `TOPAS.INC`, `ATMSCAT.CPP`): they are licensed with TOPAS. @@ -247,15 +152,15 @@ PowderLine follows the NSLS-II policy for AI coding assistants: - A human is responsible for every change, reviews every line, and confirms that the tests passed. AI output gets no shortcut through review. -- Agents work on a feature branch, commit **unsigned** if they commit at all, - and **never** sign, push, tag, force-push or rewrite published history; the - maintainer signs and pushes. +- Agents work on a feature branch and commit unsigned, if they commit at all. + They never push, tag, force-push or rewrite published history; the + maintainer pushes. - Commit messages follow Conventional Commits and end with one trailer line per model that contributed: `Assisted-by: :` (e.g. `Assisted-by: claude-code:claude-opus-5-5`). Never `Co-authored-by` for an AI, never `Signed-off-by` on a human's behalf. - Never put secrets, tokens or credentials in the repository or in a - prompt; `.powderline_config.yaml` is gitignored for that reason. + prompt. - Re-run every reported test count yourself before stating it, and compare against the baseline rather than assuming a failure is new. - Before calling documentation complete, grep for the old wording. diff --git a/CHANGELOG.md b/CHANGELOG.md index 416486b..d9f795c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and PowderLine versions follow [Semantic Versioning](https://semver.org/). The recipe **schemas** are versioned separately from the package and have their -own history, [`docs/SCHEMA_HISTORY.md`](docs/SCHEMA_HISTORY.md). +own history, [`docs/SCHEMA_HISTORY.md`](docs/reference/versions.md). ## [Unreleased] @@ -14,15 +14,15 @@ own history, [`docs/SCHEMA_HISTORY.md`](docs/SCHEMA_HISTORY.md). PowderLine 0.2.0 is a clean break from the unified recipe schema 0.26.0: a recipe names one engine and is written in that engine's own conventions, and nothing is translated between engines. Recipes written for 0.1.x do not -validate; see the [migration guide](docs/migration-0.1-to-0.2.md). Known -limitations are listed in the [known-issues register](docs/known_issues.md). +validate; see the [migration guide](docs/development/migrating-from-0.1.md). Known +limitations are listed in the [known-issues register](docs/development/known-issues.md). ### Highlights - **One recipe, one engine:** `gsasii.*`, `topas.*` and `easydiffraction.*` recipes, each in its engine's own names and units, through one API and one CLI. - **New schemas, starting at 0.1.0:** a core schema plus one schema per engine - replace the single 0.1–0.26.0 schema ([schemas](docs/schemas.md)). + replace the single 0.1–0.26.0 schema ([schemas](docs/engines/index.md)). - **Validation states what the engine computes:** symmetry, engine floors and silent defaults become errors or structured warnings, never silent changes. - **Uniform fit statistics:** `rwp`, `r_exp`, `gof`, `chi2_red` are computed by @@ -34,7 +34,7 @@ limitations are listed in the [known-issues register](docs/known_issues.md). ### Upgrading -- Read the [migration guide](docs/migration-0.1-to-0.2.md): a 0.26.0 recipe +- Read the [migration guide](docs/development/migrating-from-0.1.md): a 0.26.0 recipe (`GSASII_Rietveld` / `GSASII_SPF`) is refused at `schema_name`. - Convert gsasii recipes with `python scripts/convert_recipe_026.py IN.json -o OUT.json`; it reports every change and stops where 0.1.1's refinement cannot @@ -52,10 +52,10 @@ limitations are listed in the [known-issues register](docs/known_issues.md). - Engine schemas `gsasii.rietveld` / `gsasii.spf` (GSAS-II 5.7.9), `topas.rietveld` / `topas.spf` (TOPAS 6) and `easydiffraction.rietveld` - (easydiffraction 0.21.1), on a shared core phase block ([fields](docs/fields.md)). + (easydiffraction 0.21.1), on a shared core phase block ([fields](docs/reference/fields.md)). - Declared schema compatibility: each gateway states the core and engine schema versions it accepts, `validate()` enforces them, and `powderline.support_matrix()` - lists them ([support matrix](docs/support-matrix.md)). + lists them ([support matrix](docs/reference/versions.md)). - `powderline.validate()`, engine-free, returning `ValidationResult(recipe, warnings)`; an invalid recipe raises pydantic's `ValidationError` listing every problem at its field path. @@ -74,7 +74,7 @@ limitations are listed in the [known-issues register](docs/known_issues.md). with value and ESD, by GSAS-II's own name. - Ionic `scattering_types` (CIF spelling, e.g. `La3+`) per atom label in the gsasii and easydiffraction (`cryspy`) phase blocks, checked against each - engine's ion table ([input guidance](docs/input-guidance.md); EB-94–EB-96). + engine's ion table ([input guidance](docs/engines/index.md); EB-94–EB-96). - March–Dollase `preferred_orientation` (`hkl`, `ratio`) in the gsasii and easydiffraction (`cryspy`) phase blocks (EB-97, EB-98). - TOPAS accepts ionic `scattering_types` (TOPAS 6's table, 113 ions, written as @@ -95,16 +95,16 @@ limitations are listed in the [known-issues register](docs/known_issues.md). - Examples: per-engine recipes under `examples//{topas,easydiffraction}/`, `example_LaB6_temperature_series` (sequential refinement), `example_LaB6_ionic` and `example_LaB6_textured`, with easydiffraction references. -- Documentation: [schemas](docs/schemas.md), [fields](docs/fields.md), [input - guidance](docs/input-guidance.md), [engine behaviour](docs/engine-behavior.md), - [cross-engine comparison](docs/cross-engine-comparison.md), [policies](docs/policies.md). -- Engine citations in `CITATION.cff` and on the [citing page](docs/citing.md) (#12). +- Documentation: [schemas](docs/reference/outputs.md), [fields](docs/reference/fields.md), [input + guidance](docs/concepts/validation.md), [engine behaviour](docs/engines/index.md), + [cross-engine comparison](docs/concepts/comparing-engines.md), [policies](docs/development/policies.md). +- Engine citations in `CITATION.cff` and on the [citing page](docs/about/citing.md) (#12). ### Changed - The recipe schema is split: the single 0.1–0.26.0 schema is replaced by a core schema and one schema per engine, each starting at **0.1.0** and told - apart by `schema_name`; the 0.x bump rule is in [policies](docs/policies.md). + apart by `schema_name`; the 0.x bump rule is in [policies](docs/development/policies.md). - A recipe names its engine and workflow in `schema_name` (`.`) and states that engine's own terms and units; unknown keys are errors at every level. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b51b213..c10ccca 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,76 +1,37 @@ # Contributing to PowderLine -Thanks for your interest! This page is the map; the detail lives in the linked -guides. +Thanks for your interest. This page is the short version. The full guide is +the [Development section](https://nsls2.github.io/PowderLine/development/) of +the docs: architecture, setup, tests, the docs build, adding an engine, and +the policies. -## Dev setup +## Set up ```bash -git clone https://github.com/NSLS2/PowderLine.git -cd PowderLine -pixi install -e dev # every engine + docs/test tooling (GSAS-II is built as a conda package) -pixi run test # full suite (dev env) — run it before and after your change -pixi run docs # Sphinx with -W; zero warnings is the standard +pixi install --frozen -e dev # every engine + docs and test tooling +git config core.hooksPath scripts/hooks # the pre-commit guard, once per clone +pixi run test # the engine-free suites ``` -The default environment is core-only (it validates recipes for every engine -and runs none); `pixi install -e gsasii` / `-e easydiffraction` install one -engine. After pulling a change to `pyproject.toml`'s entry points, run -`pixi reinstall powderline --all`. - -[pixi](https://pixi.sh) is the only tool you install yourself. See the -[README](README.md) for platforms and the [Quickstart](docs/quickstart.md) for -a first refinement run. - -## Where things are - -- **[docs/DEVELOPMENT.md](docs/DEVELOPMENT.md)** — developer guide: - architecture (core + gateways), the schema layer, **how to add a refinement - engine or a capability**, GSAS-II integration, workflow. -- **[docs/policies.md](docs/policies.md)** — what binds every change: - versioning and declared compatibility, deliberate behaviour changes, the - frozen GSAS-II modules, engine-version pins, review, "ask, don't - assume". -- **[docs/engine-behavior.md](docs/engine-behavior.md)** — what each engine - does with a recipe and PowderLine's policy toward it. -- **[docs/known_issues.md](docs/known_issues.md)** — the known-issues register - (`KI-NN` ids, referenced from code and docs): check it before "fixing" a - quirk. See also [docs/regression-tolerance.md](docs/regression-tolerance.md) - (why numeric comparisons are tolerance-based) and - [docs/cross-platform-guide.md](docs/cross-platform-guide.md). -- **[examples/CONTRIBUTING_EXAMPLES.md](examples/CONTRIBUTING_EXAMPLES.md)** — - how to contribute a new example recipe (examples drive schema evolution). - -## Design records - -Maintainers keep design records in a separate private repository. -`_dev/` is reserved and gitignored; never commit it. - -Dev-doc scaffold paths (`docs/superpowers/`, `docs/plans/`, `docs/specs/`, -`docs/dev/`), `.gpx` binaries, `fort.*` scratch, and `_dev/` are all rejected by -the pre-commit guard so they never reach `main`. +[pixi](https://pixi.sh) is the only tool you install yourself. ## Ground rules -- Branch off `main`; keep commits reviewable and self-contained. -- Enable the pre-commit guard once per clone: - `git config core.hooksPath scripts/hooks`. It blocks accidental commits of - data files outside the sanctioned dirs, `.gpx` binaries, and the private - `_dev/` workspace. -- Run `pixi run test` before opening a PR. Some regression cases need a working - GSAS-II refinement — compare failures against a baseline run, don't assume - they're yours. +- Branch off `main` and keep commits reviewable and self-contained. +- Run the tests before and after your change, and compare against `main`: + some GSAS-II regression cases can fail for environment reasons. +- Keep `pixi run docs` free of warnings. After editing a schema or a gateway's + declarations, regenerate the field tables (`pixi run fields-check` tells you + when they are stale). - The frozen GSAS-II modules (`src/powderline/gateways/gsasii/{helpers,project,setters,executors,extractors,constraints}.py`) - change only through the maintainers' signed-off record of frozen-code - edits; after such an edit run `pixi run update-code-hash`. Restart the GSAS-II server after - editing anything it runs (`pixi run -e gsasii gsas-server restart`). -- After editing a schema module or a gateway's version declarations, run - `PYTHONPATH=src python scripts/gen_schema_field_tables.py` and commit the - regenerated `docs/_generated/` fragments. -- If a decision is not written down (policies page, schema history), ask the maintainer rather than inferring it. -- `examples/**/output/` is tracked on purpose (regression references) — don't - regenerate it casually. + change behaviour only with the maintainers' explicit consent. Comment and + docstring edits are fine. After any edit, run `pixi run update-code-hash`. +- `examples/**/output/` is tracked on purpose (regression references). It + moves only as a deliberate, reviewed change. +- If a decision is not written down, ask the maintainers rather than infer + it. +- AI-assisted commits end with an `Assisted-by: :` trailer. - Questions and bug reports: open a GitHub issue. ## Changelog diff --git a/README.md b/README.md index dd32838..abd60c2 100644 --- a/README.md +++ b/README.md @@ -1,258 +1,137 @@ # PowderLine -[![Docs](https://img.shields.io/badge/docs-live-blue)](https://nsls2.github.io/PowderLine/) - -Automated powder X-ray diffraction refinement. You describe a refinement in a -JSON "recipe"; PowderLine runs it through the refinement engine the recipe -names — [GSAS-II](https://gsasii.github.io/), Bruker TOPAS 6, or the -open-source [easydiffraction](https://easydiffraction.org) — and returns -standardized result tables. - -**Documentation:** [nsls2.github.io/PowderLine](https://nsls2.github.io/PowderLine/) -· **New here? Start with the [Quickstart](docs/quickstart.md).** - -PowderLine gives you a programmatic, reproducible way to run Rietveld refinements -and single-peak fits without hand-driving the underlying refinement software. - -- **One recipe = one engine.** A recipe's `schema_name` names the engine and - workflow (`gsasii.rietveld`, `gsasii.spf`, `topas.rietveld`, `topas.spf`, - `easydiffraction.rietveld`) and is written in that engine's own - conventions and units; nothing is translated between engines. -- **Refine what you need:** unit cell, atom coordinates, occupancies, isotropic and - anisotropic displacement parameters, background, peak broadening and - March–Dollase preferred orientation, with ionic scattering types per atom — - each parameter individually flagged, and checked against the space-group - symmetry before any engine runs. -- **Honest statistics:** every engine returns the same uniform Rwp / Rexp / - GoF over the same data points, with the engine's own values kept alongside. -- **Strict validation, engine-free:** a recipe validates in a core-only - environment; every problem is reported at its field, and engine quirks are - refused or warned about, never papered over. -- **Standardized outputs:** fixed-filename CSV/TXT reports plus each engine's - native artifacts (the GSAS-II `.gpx`, the TOPAS `.inp` and output files). - -## 1. Install +**A complete powder diffraction refinement, as structured data that any +person, script or machine can write, check and run.** + +[Documentation](https://nsls2.github.io/PowderLine/) · +[Get started](https://nsls2.github.io/PowderLine/getting-started/) · +[How it works](https://nsls2.github.io/PowderLine/concepts/how-powderline-works.html) · +[Paper](https://doi.org/10.48550/arXiv.2608.17009) + +## Why PowderLine + +Rietveld refinement extracts structure, phase fractions and microstructure from +powder data. Doing it reliably takes expertise in a particular program, and +doing it at scale usually means custom code written for one project. + +No refinement program offered a single, validated, machine-writable +description of a whole refinement. PowderLine provides one: the **recipe**. + +## What it does + +```mermaid +flowchart LR + subgraph drivers["Who drives it"] + direction TB + D1[GUIs] + D2[Notebooks and scripts] + D3[Automated pipelines] + D4[Autonomous experiments] + D5[LLM agents] + end + subgraph pl["PowderLine"] + direction TB + R["Recipe
structured, versioned
data · model · instrument · flags · metadata"] + V{{"Validate
schema + symmetry"}} + X{{Run}} + R --> V --> X + end + subgraph engines["Refinement engines"] + direction TB + E1[GSAS-II] + E2[TOPAS] + E3[easydiffraction] + end + subgraph out["Results"] + direction TB + O1["Tables and uniform statistics"] + O2["Output recipe
refined values written back"] + O3["Provenance
run IDs · versions · your metadata"] + end + drivers --> R + X <--> engines + X --> out + O2 -. next step .-> R +``` -PowderLine uses [pixi](https://pixi.sh) for reproducible conda environments. -The **default environment is core-only** (schema validation, the CLI, no -engine); each engine lives in its own environment: +1. **Describe.** A recipe is a structured document holding the data, the + structural model, the instrument, what to refine and any metadata you + attach. In Python it is a dictionary. On disk or over the wire it is + JSON. It can come from a file, a database record or a live data stream. +2. **Validate.** PowderLine checks the recipe against a versioned schema and + against the crystallography: the cell against the space group, atoms on + special positions, symmetry-tied parameters. No engine needs to be + installed. Each problem is reported at its field, with the fix. +3. **Run.** PowderLine hands the recipe to the engine it names (GSAS-II, + TOPAS or easydiffraction) and returns the same structured results whichever + engine ran. +4. **Chain.** Every result includes an *output recipe*, with the refined + values written back and provenance attached. That is the next step's + input: calibrant → sample → temperature series. + +## Why it matters + +- **One interface for every driver.** A GUI, a notebook, a beamline pipeline, + an autonomous experiment and an LLM agent all drive PowderLine the same + way, with the same recipe, and get the same result. +- **Modular.** PowderLine sits between data and decisions. It takes data + from your reduction step and hands structured results to whatever comes + next: plotting, databases, decision-making for the next measurement. +- **Machine-checkable.** Validation names every problem and how to fix it, so + a program or an LLM can write a recipe, check it and correct it without a + person in the loop. +- **Traceable.** Your metadata (sample, measurement, data IDs) travels with + the recipe into the results, and every run records what produced it. Data, + analyses and their order stay linked. +- **Comparable.** Every engine reports Rwp, Rexp and GoF computed the same way + over the same points. + +## Try it + +You need [pixi](https://pixi.sh) and git. ```bash git clone https://github.com/NSLS2/PowderLine.git cd PowderLine -pixi install # core only: validate recipes, no engine -pixi install -e gsasii # + GSAS-II (built as a conda package by pixi-build) -pixi install -e easydiffraction # + easydiffraction (Python >= 3.12) -pixi install -e all # every engine -# fresh checkout: add --frozen (e.g. `pixi install --frozen -e gsasii`) so the committed lock is used as is -``` - -Platforms: `linux-64`, `win-64`, `osx-arm64`. TOPAS has no Python package: -`topas.*` recipes validate everywhere and their input files are written -everywhere, but running them needs a Windows machine with TOPAS 6 (`tc.exe`). - -## 2. Getting started - -Run the bundled LaB6 calibrant example with GSAS-II: - -```bash -pixi run -e gsasii kicker examples/example_LaB6/input.json -``` - -Results are written next to the recipe in `examples/example_LaB6/output/`: - -| File | What it is | -|---|---| -| `refined_parameters.csv` | Every refined parameter with its ESD (9-column table) | -| `LaB6_unit_cell_report.csv` | Per-phase unit-cell parameters + ESDs | -| `LaB6_peak_list_report.csv` | Reflection list (hkl, d-spacing, 2θ, intensity) | -| `fit_profile.txt` | Observed / calculated / background / difference | -| `output_recipe.json`, `parameter_dispositions.csv` | The recipe with the refined values written back, and how each value came about | -| `dummy.lst` | Human-readable GSAS-II refinement log (Rwp, parameter tables) | -| `dummy.gpx` | GSAS-II project — reopenable in the GSAS-II GUI | - -PowderLine removes its own fixed-name files from the output directory before -each run, so a reused directory holds one run's results. A failed run raises, -and its directory holds the engine's own files plus `failure.json` (the reason, -the message and the engine's last console lines) — never a result table. - -The same case as a TOPAS and an easydiffraction recipe sits in -`examples/example_LaB6/topas/` and `examples/example_LaB6/easydiffraction/`. - -> `examples/example_template/` holds fill-in-the-blanks skeletons for the -> three engines, not runnable recipes — copy one to start your own. - -**Bringing your own CIF, calibrant and data files?** PowderLine has no -built-in recipe builder; [Building a recipe](docs/input-guidance.md#building-a-recipe) -walks through writing one and validating it. - -## 3. Ways to use PowderLine - -**Command line** — one command for every engine; the recipe's `schema_name` -decides where it runs: - -```bash -pixi run kicker --validate-only # check a recipe; needs no engine -pixi run -e gsasii kicker # run a gsasii.* recipe -pixi run -e easydiffraction kicker # run an easydiffraction.* recipe -pixi run kicker --topas-dir C:/TOPAS6 --topas-version 6 # run a topas.* recipe (tc.exe on Windows) -pixi run -e gsasii doctor # what this environment can do (gateways, engines, server) +pixi install --frozen -e gsasii +pixi run -e gsasii powderline examples/example_LaB6/input.json ``` -After `pixi reinstall powderline` the same program is the installed command -`powderline …`. Flags: `--output DIR` (default: `output/` next -to the recipe), `--verbose`, `--validate-only`; gsasii `--use-server` / -`--no-server` / `--server-timeout SECONDS` (default: no limit); topas -`--topas-dir DIR` / `--topas-version V` / -`--topas-timeout SECONDS` (default 3600; 0 = no limit). A flag given -for a recipe of another engine is an error. +The results are in `examples/example_LaB6/output/`. +[Your first refinement](https://nsls2.github.io/PowderLine/getting-started/first-refinement.html) +walks through them. -**Python API** — load a recipe, run it, get pandas DataFrames back: +## Engines -```python -import powderline, tempfile -from pathlib import Path +| Engine | Workflows | Runs on | +|---|---|---| +| GSAS-II 5.7.9 | Rietveld, single-peak fitting | Linux, Windows, macOS | +| TOPAS 6 | Rietveld, single-peak fitting | Windows with a TOPAS licence (input files are written anywhere) | +| easydiffraction 0.21.1 | Rietveld | Linux, Windows, macOS | -recipe = powderline.load_recipe_asset(Path("examples/example_LaB6/input.json")) +## Cite -# Engine-free check: every problem at its field path; warnings for defaults applied -checked = powderline.validate(recipe) -print(checked.recipe.schema_name, checked.warnings) - -# run() writes its file reports into the directory you give it; the engine is -# the one the recipe names, so this needs the gsasii environment: -result = powderline.run(recipe, Path(tempfile.mkdtemp())) - -print(result["rwp"]) # PowderLine's uniform Rwp (%) -print(result["engine_details"]) # GSAS-II's own statistics, parameters requested vs varied -print(result["refined_parameters"]) # DataFrame of refined values + ESDs -``` - -The returned dict holds the fit metadata (`status`, `converged`, `rwp`, `r_exp`, -`gof`, `chi2_red`, `simulation_mode`, `engine_details`, `warnings`, elapsed time, -which execution path ran, the output recipe with its parameter dispositions; a -failed run raises instead of returning) plus every -tabular report in memory — the refined-parameter table with ESDs, the fit -profile, per-phase unit-cell and peak-list tables (dicts keyed by phase -name), and the SPF peak tables when single-peak fitting. Everything the -CSV/TXT files contain, without touching disk yourself. - -The public API (`powderline.__all__`) is `run`, `validate`, -`ValidationResult`, `load_recipe_asset`, `support_matrix` and the exceptions -`PowderLineError`, `GatewayNotInstalledError`, `EngineNotAvailableError`, -`EngineVersionError`, `EngineExecutionError`. `import powderline` imports no -engine. - -**The GSAS-II server (used by default for gsasii recipes)** — refinements go -through a persistent server that keeps GSAS-II resident in memory -(auto-started as needed), a large speedup across many refinements. Manage it -directly, or opt out per run with `--no-server`: - -```bash -pixi run -e gsasii gsas-server status # start | status | stop | restart | logs -pixi run -e gsasii kicker --no-server # force an isolated in-process run -``` - -The server is per-user: it listens on loopback only and accepts only requests -signed with a secret token kept in a directory only you can read; your client -in turn only accepts results signed with that token, so other users on the -same machine can neither use your server nor impersonate it. To switch it off -— e.g. on a shared workstation — set `POWDERLINE_NO_SERVER=1`: default -(`auto`) runs then go in-process, explicit server mode (`--use-server` / -`execution_mode='server'`) fails, and nothing is auto-started. - -## 4. Refinement engines - -| Engine | Schemas | Environment | Notes | -|---|---|---|---| -| **GSAS-II 5.7.9** | `gsasii.rietveld`, `gsasii.spf` | `gsasii` | The reference engine; the committed examples carry its outputs as regression references. | -| **TOPAS 6** (Bruker) | `topas.rietveld`, `topas.spf` | any (writing the INP); Windows + `tc.exe` (running) | The recipe states `engine_version: "6"`; the run needs the installed version and location declared (`--topas-version 6 --topas-dir DIR`, or `topas.version` / `topas.dir` in `.powderline_config.yaml`); PowderLine never guesses where `tc.exe` is. | -| **easydiffraction 0.21.1** | `easydiffraction.rietveld` | `easydiffraction` | The recipe states the calculator (`cryspy` or `crysfml`); no single-peak fitting. | - -The structural part of a recipe (space group, cell, atoms) is identical in -every engine; the instrument, background and controls are each engine's -own, with its own units. The -[cross-engine comparison](docs/cross-engine-comparison.md) refines the same -two data sets with every engine and explains why the numbers differ: -**compare lattice parameters, and read Rwp as a property of the model, not -of the engine.** What each engine does with your recipe, and what PowderLine -refuses on its behalf, is on the -[engine behaviour page](docs/engine-behavior.md). - -For the engine architecture and how to add a backend, see -[Adding a refinement engine](docs/DEVELOPMENT.md#adding-a-refinement-engine). -Maintainers keep design records in a separate private repository. - -## Documentation - -- **[Quickstart](docs/quickstart.md)** — clone → install → run → read the output. -- **[Getting Started](docs/getting-started.rst)** — recipe structure, output - files, and the Python API. -- **[Recipe schemas](docs/schemas.md)**, **[Fields and features](docs/fields.md)**, - **[Input guidance](docs/input-guidance.md)** — the five schemas, every field - with its unit, how to supply data / structures / instruments. -- **[Engine behaviour notes](docs/engine-behavior.md)** — what each engine - does, and PowderLine's policy toward it. -- **[Troubleshooting](docs/TROUBLESHOOTING.md)** — common errors and fixes. -- **[Citing PowderLine](docs/citing.md)** — PowderLine, the engines and the - libraries behind them. -- **[Migration guide](docs/migration-0.1-to-0.2.md)** — from PowderLine 0.1.1 - (unified schema 0.26.0) to 0.2.0 (core + engine schemas 0.1.0). -- **[Schema History](docs/SCHEMA_HISTORY.md)** and the - **[support matrix](docs/support-matrix.md)** — schema versions and declared - compatibility. -- **[Using PowderLine as a library](docs/integration-guide.md)** — install it into - another project. -- **[Developer Guide](docs/DEVELOPMENT.md)** and **[Policies](docs/policies.md)** - — architecture, how to add an engine, versioning, frozen code, review - gates. See also the [known-issues register](docs/known_issues.md) - (`KI-NN` ids referenced from code and docs). - -## Citation - -If you use PowderLine in your research, please cite the preprint: +PowderLine is described in a paper accepted for publication in the *Journal +of Applied Crystallography*. Until it appears, please cite the preprint: > A. A. Corrao, J. A. Perez, J. D. Langhout, M. M. Butala, T. A. Caswell, and -> D. Olds. *PowderLine: a programmatic powder diffraction analysis application.* -> arXiv:2608.17009 (2026). +> D. Olds. *PowderLine: a programmatic powder diffraction analysis +> application.* arXiv:2608.17009 (2026). > [doi:10.48550/arXiv.2608.17009](https://doi.org/10.48550/arXiv.2608.17009) -Machine-readable metadata is in [`CITATION.cff`](CITATION.cff). - -### Cite the refinement engine too - -PowderLine drives an underlying refinement engine: the one your recipe's -`schema_name` names. Please cite the engine you used in addition to -PowderLine. +**Please cite the engine you used too:** -- **GSAS-II** (`gsasii.*` recipes; GSAS-II 5.7.9 in the `gsasii` environment): - > B. H. Toby and R. B. Von Dreele. *GSAS-II: the genesis of a modern - > open-source all purpose crystallography software package.* J. Appl. Cryst. - > **46**, 544–549 (2013). - > [doi:10.1107/S0021889813003531](https://doi.org/10.1107/S0021889813003531) - - PowderLine drives GSAS-II through its scripting interface, so please also cite: - > J. H. O'Donnell, R. B. Von Dreele, M. K. Y. Chan, and B. H. Toby. *A - > scripting interface for GSAS-II.* J. Appl. Cryst. **51**, 1244–1250 (2018). - > [doi:10.1107/S1600576718008075](https://doi.org/10.1107/S1600576718008075) -- **TOPAS** (`topas.*` recipes; TOPAS 6): - > A. A. Coelho. *TOPAS and TOPAS-Academic: an optimization program - > integrating computer algebra and crystallographic objects written in C++.* - > J. Appl. Cryst. **51**, 210–218 (2018). - > [doi:10.1107/S1600576718000183](https://doi.org/10.1107/S1600576718000183) -- **easydiffraction** (`easydiffraction.*` recipes): the easydiffraction - project asks you to cite the specific version you used, through its - [Zenodo archive](https://doi.org/10.5281/zenodo.16806521) (all versions). - PowderLine's `easydiffraction` environment installs version 0.21.1: - > A. Sazonov, E. Lindgren, P. Rozyczko, and M. Seth. *easydiffraction - > 0.21.1.* Zenodo (2026). - > [doi:10.5281/zenodo.22905049](https://doi.org/10.5281/zenodo.22905049) +| Engine | Cite | +|---|---| +| GSAS-II | Toby & Von Dreele, *J. Appl. Cryst.* **46**, 544 (2013), [doi](https://doi.org/10.1107/S0021889813003531); O'Donnell et al., *J. Appl. Cryst.* **51**, 1244 (2018), [doi](https://doi.org/10.1107/S1600576718008075) | +| TOPAS | Coelho, *J. Appl. Cryst.* **51**, 210 (2018), [doi](https://doi.org/10.1107/S1600576718000183) | +| easydiffraction | easydiffraction 0.21.1, Zenodo (2026), [doi](https://doi.org/10.5281/zenodo.22905049), plus the calculator your recipe names (CrysPy or CrysFML) | - Please also cite the calculator your recipe names in - `refinement_controls.calculator`: CrysPy (`cryspy`) or CrysFML (`crysfml`). - The references are on the [citing page](docs/citing.md), which also covers - gemmi, the library behind PowderLine's space-group checks. +Full references, including the calculators and gemmi, are on +[Citing PowderLine](https://nsls2.github.io/PowderLine/about/citing.html). +Machine-readable metadata is in [`CITATION.cff`](CITATION.cff). -## License +## Contributing and licence -BSD 3-Clause — see [LICENSE](LICENSE). +See the [developer guide](https://nsls2.github.io/PowderLine/development/). +BSD 3-Clause licence, see [LICENSE](LICENSE). diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md deleted file mode 100644 index 9015396..0000000 --- a/docs/DEVELOPMENT.md +++ /dev/null @@ -1,719 +0,0 @@ -# PowderLine Developer Guide - -This guide provides context for developers working on PowderLine (and for AI -coding assistants used alongside them). Read this first to understand the -architecture, the design decisions, and the development workflows. The rules -that bind every change (versioning, deliberate changes, the frozen GSAS-II -code, the engine pins, review gates, "ask, don't assume") are on the -[policies page](policies.md); this guide shows where things are and how to -work on them. - -## Project Overview - -**What is PowderLine?** -PowderLine automates crystallographic refinement from a declarative JSON -"recipe" and drives interchangeable refinement engines — GSAS-II, Bruker -TOPAS 6 and the open-source easydiffraction — through one API and one CLI. -A recipe names its engine and workflow in `schema_name` and is written in -that engine's own conventions; PowderLine validates it strictly, hands it to -the engine's **gateway**, and returns standardized result tables with uniform -fit statistics. The goal is to make powder diffraction refinements -reproducible, automatable, and accessible. - -**Scope:** -- One CLI (`powderline `) and one Python API (`powderline.run`, - `powderline.validate`) for every engine; the recipe decides where it runs -- A **core schema** (the shared frame and structural models) plus one - **engine schema** per gateway, all pydantic, `extra='forbid'`, engine-free -- Three gateways under `src/powderline/gateways/`, discovered through entry - points, each split into an engine-free schema layer and a lazily imported - runtime layer -- Single-phase and multi-phase Rietveld refinement; single peak fitting - (GSAS-II, TOPAS) -- Standard output reports plus each engine's native artifacts; uniform - Rwp / Rexp / GoF computed once in core, the engine's own values alongside -- Example-driven design: real refinement cases drive the schemas and are the - regression references - -**Design Philosophy:** -- **Example-driven**: don't build features speculatively; add them when real examples need them -- **Schema-first**: every rule a recipe must obey lives in the schema models, so validation - reports every problem at its field before any engine runs -- **Honest**: an engine's silent default, floor or conversion is either refused at - validation (with the value to write) or reported as a structured warning; never papered over -- **Engine-native**: no translation between engines; each recipe states the numbers its engine computes with -- **Reproducible**: the same recipe produces the same refinement; the references are tracked - -## Architecture (`src/powderline/`) - -### Core (engine-free) - -| Module | Role | -|---|---| -| `__init__.py` | the public API, `__all__`: `run`, `validate`, `ValidationResult`, `load_recipe_asset`, `support_matrix`, the exceptions. `import powderline` imports no engine | -| `api.py` | `validate()` and `run()`: resolve the gateway from `schema_name` (`.`) through the registry and hand the recipe over; `ValidationResult` | -| `kicker.py` | the one CLI (`powderline `; `pixi run kicker` runs the same `main`): loads the recipe, refuses templates, maps the engine flags onto the gateway's run options, prints the summary and the structured warnings | -| `registry.py` | gateway discovery from the `powderline.gateways` entry-point group, lazy loading, `Gateway` dataclass (`name`, `capabilities`, `validate`, `run`), `register_gateway` escape hatch, the engine-version check | -| `schema_core.py` | core schema 0.1.0: `CoreRecipe` (frame), `RefinableParameter` / `BoundedRefinableParameter`, `XRDData`, `FitRange`, `ChebyshevBackground`, `UnitCell`, `Atom`, `UanisoTensor`, `Phase[P]` (the phase block with every symmetry rule), `CoreFloat`/`CoreInt`/`CoreName`, `collect_warnings`, `parameters_requested`, `fit_limits_on_data`; the engine schema building blocks: `EngineRecipe`, `recipe_validator`, `refined_flags`, `custom_error` / `raise_problems`, `omit_absent`, `absent_not_null`, `unique_phase_names`, `fit_window`, `fit_range_problems`, `outside_window_problems`, `refined_flag_problems`, `unit_schema` | -| `symmetry.py` | gemmi-backed symmetry: canonical space-group names, cell tie groups and the cell check, special-position analysis, coupling groups for coordinates and Uij | -| `compat.py` | declared schema-version compatibility: `check_engine_schema_version`, `support_matrix()`, `schema_declarations()` (a gateway's `DECLARATIONS`) | -| `fitstats.py` | the uniform Rwp / Rexp / GoF / χ²_red over the fit window with positive weights | -| `reports.py` | the standardized report writers (`refined_parameters.csv`, unit-cell and peak-list reports, `fit_profile.txt`): one column order, UTF-8, LF; the fixed-name list PowderLine owns (`FIXED_OUTPUT_NAMES` / `FIXED_OUTPUT_PATTERNS`, `FAILURE_FILE`) and `clear_fixed_outputs()` | -| `recipes.py` | `load_recipe_asset` (JSON / YAML) | -| `exceptions.py` | `PowderLineError`, `SymmetryError`, `GatewayNotInstalledError`, `EngineNotAvailableError`, `EngineVersionError`, `EngineExecutionError`, `ExecutionBackendError` (both with `.reason` / `.result`), the `StructuredWarning` shape; `WARNING_CODES`, the registry of every structured-warning code with its owner and level (asserted against `src/` and the engine-behaviour table) | -| `results.py` | the run-result envelope: `ENVELOPE_VERSION`, the reason-code sets, `failure_envelope`, `generated_envelope`, `with_status_fields`, `raise_if_failed` | -| `config_loader.py` | `.powderline_config.yaml` (the `simulation_defaults` block; engine-free) | -| `_status.py` | console status glyphs | - -### Gateways (`src/powderline/gateways//`) - -Each gateway is a package with the same two layers: - -- **Schema layer** — `schema.py`: the engine schema models built on - `schema_core` (the engine's `Phase` subclass, instrument, background, - controls, payload and recipe models), the version declarations - (`ENGINE_SCHEMA_VERSION`, `SUPPORTED_ENGINE_SCHEMAS`, `REQUIRES_CORE_SCHEMA`, - exposed as `DECLARATIONS`), `SCHEMA_NAMES`, and `validate_recipe()`. Imports - no engine; this is what `powderline.validate()` runs. -- **Runtime layer** — `gateway.py` (the entry-point factory: `capabilities()`, - `validate()`, `run()`, `ENGINE_VERSION_SPEC`, and for pixi-installed engines - `ENV` / `INSTALL_COMMAND`) plus the modules that import the engine, imported - lazily from `run()`. - -| Gateway | Runtime modules | -|---|---| -| `gsasii/` (GSAS-II 5.7.9) | the **frozen** setup/execution code moved verbatim from `kicker.py` at the baseline `daace24`: `helpers.py`, `project.py`, `setters.py`, `executors.py`, `extractors.py`, `constraints.py` (code-hashed; see [policies](policies.md#frozen-gsas-ii-code)); non-frozen additions `routing.py` (one entry for in-process, server and client), `fit_report.py` (uniform statistics, `engine_details`, convergence, divergence), `sites.py` (GSAS-II's reading of each site vs core's), `report_rows.py`, `paths.py` (GSAS-II parameter names → recipe paths for the output recipe), `console.py` (the engine's console tail into a failed envelope), `scattering.py` (GSAS-II's ion table and spelling); `server.py` / `client.py` (the persistent FastAPI server and `GSASClient`: per-user, loopback, ephemeral port, signed requests and responses, `POWDERLINE_NO_SERVER=1` disables it — v0.1.2); `simulation_builder.py` (structure dict → simulation recipe; see [the builder](#the-simulation-recipe-builder)) | -| `topas/` (TOPAS 6) | `native_writer.py` (model → INP + `.xye`), `native_run.py` (run `tc.exe`, read TOPAS's files strictly, build the result; the background-only second run), `peak_list.py`, `runner.py` (find `tc.exe`, read `.powderline_config.yaml`) | -| `easydiffraction/` (0.21.1) | `native_builder.py` (model → easydiffraction `Project`; the CrysPy template rule; anisotropic atoms first), `native_run.py` (calculate or lmfit fit, the result), `space_group_table.py` / `space_group_support.py` (the 531 settings and the per-calculator verified sets), `scattering.py` (CrysPy's ion table) | - -The entry points are declared in `pyproject.toml` -(`[project.entry-points."powderline.gateways"]`); the metadata is written at -install time, so after editing that table run `pixi reinstall powderline` -(every environment has its own install: `--all`). There is deliberately no -in-code fallback (KI-13): the in-repo gateways use exactly the mechanism a -third-party engine would. - -### The execution stack - -``` -powderline / powderline.run(recipe, output_dir, **engine_options) - │ schema_name -> registry.get("") ; gateway.run(...) - ▼ -gateway.run() validates (schema layer), checks the engine version, - │ then the runtime layer: - ├─ gsasii: routing.run_refinement() in this process, or GSASClient - │ -> the FastAPI server (GSAS-II resident) -> routing.run_refinement() - │ -> executors.run_refinement() -> GSASIIscriptable - │ -> fit_report (uniform statistics) ; reports (files) - ├─ topas: native_writer -> tc.exe (twice: fit, background) -> native_run reads the files - └─ easydiffraction: native_builder -> lmfit / calculate() -> native_run - ▼ -the standardized result dict (tables as DataFrames) + the report files + native artifacts -``` - -`execution_mode` (gsasii only: `'auto'` | `'server'` | `'subprocess'`) is the -public option; `method` is the result key saying which path ran. The server -keeps GSAS-II imported in one process (~2.6 s import amortized); it caches -code, so **restart it after editing anything it runs** -(`pixi run -e gsasii gsas-server restart`). - -## The schema layer - -### Shape of a recipe - -``` -schema_name "." -core_schema_version "0.1.0" -engine_schema_version "0.1.0" -metadata free-form, echoed, never read (<= 1 MiB) -payload the engine schema's payload model -``` - -Core owns the frame, `xrd_data`, `fit_range`, `ChebyshevBackground` and the -**phase block** (`space_group`, `unit_cell`, `atoms`), generic over the -parameter type `P` (`RefinableParameter` for gsasii, `BoundedRefinableParameter` -for topas and easydiffraction). The engine schema owns everything else -(instrument, background placement, controls, engine phase fields such as -`scale` and `peak_broadening`) and may add checks, but can never redefine, -skip or rewrite core's (guarded when the class is defined: no reuse of a -core field, validator or private-attribute name, no `'before'`/`'wrap'` -validator on a core field, `extra='forbid'` kept; `tests/test_phase_canary.py` -runs core's must-fail recipes through every engine phase model). - -User-facing rules: [schemas](schemas.md), [fields](fields.md) (generated -tables), [input guidance](input-guidance.md), [SCHEMA_HISTORY](SCHEMA_HISTORY.md). - -### Validation model - -- Every recipe check is a pydantic validator raising `ValueError` (or - `SymmetryError`), so an invalid recipe raises **one** `pydantic.ValidationError` - listing every problem at its field path. There is no PowderLine wrapper - exception for invalid recipes. -- **Validation never changes a recipe.** The validated model holds exactly the - recipe's values plus documented defaults and the symmetry-derived values; a - value an engine would silently replace (a floor, a snapped coordinate, a - limit) is an **error naming the value to write**, never an adjustment. -- **Structured warnings** (`{code, message, field_path[, level]}`; `level` absent = `"warning"`, or `"info"`) are collected once - from the validated model (`collect_warnings`) and travel with it: a default - applied, a symmetry remark, a unit conversion the gateway will make. They - never change what the recipe means, so re-validating reproduces them. -- Engine quirks that reach a user are recorded in the maintainers' - engine-behaviour register and rendered on the - [engine behaviour page](engine-behavior.md); a schema rule that exists - because of a quirk cites its `EB-nn` in the error message. - -## Adding a refinement engine - -The gateway seam is the contract the refactor exists to provide. A new engine -is a package `src/powderline/gateways//` (in-repo) or a separate -distribution (out of tree) that provides: - -1. **A schema module** (`schema.py`), engine-free: - - the engine's `Phase` subclass (`class MyPhase(Phase[BoundedRefinableParameter])` - or `Phase[RefinableParameter]` if the engine has no usable bounds), adding - the engine's phase fields with their own validator names and `'after'` - validators only; - - instrument, background and controls models in the **engine's native - terms and units**, every quantity carrying `json_schema_extra={"unit": ...}` - (the [fields page](fields.md) is generated from it), `extra='forbid'`, - no `null` for a block that may be left out, and **no default that - replaces an engine behaviour silently** (either the value is required, or - the default is documented and reported by a warning); - - the recipe model(s) subclassing `powderline.schema_core.EngineRecipe` - with `schema_name` narrowed to a `Literal` and two class variables, - `gateway_name` and `schema_declarations = DECLARATIONS`, where - `DECLARATIONS = compat.schema_declarations(engine_schema_version="0.1.0", - supported_engine_schemas="==0.1.0", requires_core_schema="==0.1.0")` - (widened explicitly later, never by range); `EngineRecipe` runs the - version check; - - `validate_recipe = schema_core.recipe_validator({: , ...}, - engine="", title="Recipe")`: accepts a dict or an - already-validated model (idempotent) and reports an unknown - `schema_name` at `schema_name`; - - every engine rule as a validator: refuse what the engine cannot express - (settings, features) and what it would silently alter (floors, limits), - naming the value to write; apply the three shared value rules - (wavelength > 0, scale per engine, peak positions inside the window) and - the fit window on data points (`fit_limits_on_data`). The shared pieces - are core building blocks, not copies: `absent_not_null` for a - block that may be left out but never be `null`, `omit_absent` in the - wrap serializer that drops it from dumps, `unique_phase_names`, - `fit_range_problems` / `fit_window` / `outside_window_problems` / - `refined_flag_problems` for the payload rules, `refined_flags`, - `custom_error` + `raise_problems` to report every problem at once; - - every warning code the engine emits registered in - `powderline.exceptions.WARNING_CODES` (owner, level) and listed in the - engine-behaviour table; a core test holds the three in agreement. -2. **A gateway module** (`gateway.py`) exporting `get_gateway()` → - `registry.Gateway(name, capabilities, validate, run)`: - - `capabilities()` returns at least `name`, `workflows` (the schema names), - `engine_version_spec` (PEP 440) and the schema `DECLARATIONS`; - - `validate(recipe, *, verbose=False)` (exactly these keywords) returns the - validated model; - - `run(recipe, output_dir, *, verbose=False, validate_only=False, - allow_unsupported_engine_version=False, **own_options)`: validates, checks - the installed engine version against `engine_version_spec` - (`registry.check_engine_version`; raise `EngineNotAvailableError` with - `env` and `install_command` when the engine is missing — only on a real - run, never for `validate_only`), imports the engine **lazily**, runs, and - hands `powderline.roundtrip` its refined values by recipe path (`{path: (value, esd)}` - in recipe units, plus the flagged paths it did not vary and those that ended at a - limit; `roundtrip.build_output` / `attach` do the rest: tie propagation, the - multiplicity invariant, `output_recipe`, `output_recipe_valid`, `parameter_dispositions`, - the two files) and - returns the standardized result dict with the same keys and types as the - other gateways (`status`, `converged`, `envelope_version`, `run_id`, `rwp`, `r_exp`, `gof`, `chi2_red`, - `simulation_mode`, `engine_details` with `parameters_requested` / - `parameters_varied` / `n_points`, `warnings`, `elapsed_time`, `method`, - `output_files`, `fit_profile`, `unit_cell_data`, `peak_list_data`, - `refined_parameters`, `spf_peaks`, `spf_convergence_diagnostics`; gsasii adds - `peak_fit` for a peak fit). A gateway **never returns a failed - result**: it raises `EngineExecutionError` (the engine ran and failed) or - `ExecutionBackendError` (it did not run) with a `reason` from - `powderline.results.REASONS`, or returns - `results.raise_if_failed(envelope)` on an envelope it received; the - GSAS-II server/client show the wire form (`results.failure_envelope`, - `results.with_status_fields`). Statistics come from `powderline.fitstats` over - the window's weighted points with P read from the engine after the run; - the engine's own values go under `engine_details`; divergence (a - non-finite calculated pattern) raises `EngineExecutionError`. - - report files through `powderline.reports`; native artifacts kept. -3. **An entry point** in `pyproject.toml` - (`[project.entry-points."powderline.gateways"] = "...gateway:get_gateway"`), - then `pixi reinstall powderline`. A pixi feature/environment for the - engine's dependencies if the default environment must not carry them - (every in-repo engine has one); `ENV` / `INSTALL_COMMAND` on the gateway. -4. **Tests**, in the pattern of the existing gateways: schema tests - (`test__schema.py`: every rule, every refusal, with the error - path), the engine phase model registered in `tests/test_phase_canary.py` - `ENGINE_PHASES`, an import-block test proving the schema layer and - `import powderline` load no engine module, a space-group translation test - over every setting the engine accepts (orbit / operator comparison against - gemmi), fixtures + goldens for the runtime (hand-derived native recipes - with a README of the derivations; goldens recording the engine's version), - and the result-contract test (same keys and types as the other engines). - Tests that need the engine are marked / skipped so the core-only suite - stays green. -5. **Freeze-equivalent expectations**: the engine's goldens are the - references; a change in what the engine is asked to do (the INP, the - project, the input dictionary) is proved against them, and anything the - engine does silently is written up in the engine-behaviour register with - its policy before the gateway ships. -6. **Docs**: the schemas, fields (regenerate with - `scripts/gen_schema_field_tables.py`), input-guidance and engine-behaviour - pages; a SCHEMA_HISTORY section; the support matrix regenerates itself. - -### Adding a capability to an existing engine - -The same discipline, smaller: a new field or block goes into the engine -schema with its unit, its validators and its refusals; the gateway consumes it -from the validated model (for gsasii, an **input-side read in a frozen module -is a signed-off frozen-code edit**, otherwise the read happens in the non-frozen modules); a -fixture and a golden prove it reaches the engine and does what the recipe -says; the engine-behaviour register records anything the engine does with it -silently; SCHEMA_HISTORY and the generated field tables record the field. If -the field changes what an existing recipe means, that is a deliberate change -(recorded, CHANGELOG, migration note) — and if the engine's own behaviour -is unknown, that is a question for the maintainer, not an inference. - -Worked example — March–Dollase preferred orientation: - -1. **Ground truth first.** Read and run each engine: GSAS-II's HAP - `Pref.Ori.` entry and `GetPrefOri`; easydiffraction's - `preferred_orientation` rows, which only CrysPy reads, and only under the - lowercase structure key (EB-97); CrysPy's un-normalised correction (EB-98). -2. **One core building block, engine fields.** `schema_core.MarchDollase[P]` - carries the shape and the engine-independent rules (`hkl` not all zero, - `ratio > 0`); each engine's phase model adds `preferred_orientation` in - its own parameter shape (gsasii `[value, flag]`, easydiffraction bounded - plus the required `march_random_fract`, topas bounded); topas declared the - field and refused it until its half was verified on a TOPAS 6 machine - (the `PO` macro against the Dollase formula, then the writer and the - read-back of r). -3. **The gateway reads the model.** gsasii: an additive frozen setter and its - call, two signed-off frozen-code edits with proofs (the project-dict diff shows only - `Pref.Ori.`), and the refined name `p:h:MD` in `paths.py`; - easydiffraction: the builder's row plus the `_free` bookkeeping that maps - the refined value back to the recipe path. -4. **A gate, then a demonstration.** The pass/fail gate is a formula check on a - simulated pattern (each engine's textured/untextured intensity ratio equals - Dollase's correction over the reflection's equivalents); the example - (`example_LaB6_textured`) is the demonstration that refinement recovers the - programmed value, with its caveats written down (the start at exactly 1.0 - aborts, EB-99). -5. **Record.** Fields tables regenerated, SCHEMA_HISTORY per engine, - engine-behaviour register rows, CHANGELOG, the T-list row for the TOPAS - half, the P-row. - -## GSAS-II integration notes - -### The frozen modules and the project dictionary - -The GSAS-II setup/execution code manipulates the `proj.data` dictionary of a -GSAS-II project directly (explicit control, debuggable, brittle to GSAS-II -internals). It moved verbatim out of `kicker.py` at the baseline -`daace24` (not on `main`) and is -frozen behind a code hash; input-side edits go through the maintainers' -signed-off record of frozen-code edits ([policies](policies.md#frozen-gsas-ii-code)). -The module headers still read "moved verbatim"; that record, not the headers, -is the record of the additive edits made since (ionic types in `project.py`, -the preferred-orientation setter in `setters.py`, the simulation ESDs in -`extractors.py`). -Key access patterns, for reading the code: - -```python -# Instrument parameters (list format: [current_value, value, refine_flag]); the native -# gateway builds Iparm1 from the recipe's instrument block (schema.gsasii_iparm1) -proj.data[hist.name]['Instrument Parameters'][0]['Lam'][1] = wavelength -proj.data[hist.name]['Instrument Parameters'][0]['Lam'][2] = refine_flag - -# Background -proj.data[hist.name]['Background'] = [ - [chebyshev, no_peaks, background_type], # Background metadata - {'nDebye': 0, 'debyeTerms': [], ...}, # Debye-Scherrer params - {'nPeaks': n, 'peaksList': [...]}, # Single peaks -] - -# Histogram-atom parameters (HAP): phase scale, size, microstrain -proj.data['Phases'][phase_name]['Histograms'][hist.name]['Scale'][0] = value -proj.data['Phases'][phase_name]['Histograms'][hist.name]['Size'][1][0] = value # isotropic size -proj.data['Phases'][phase_name]['Histograms'][hist.name]['Mustrain'][1][0] = value # isotropic microstrain - -# Phase parameters -proj.data['Phases'][phase_name]['General']['Cell'] = [refine_cell, a, b, c, alpha, beta, gamma, volume] -``` - -GSAS-II exposes lumped refine flags (one for the cell, `'X'` for all three -coordinates of an atom, `'U'` for all six Uij); `constraints.py` sets the -lumped flag and emits GSAS-II "Hold" constraints for the degrees of freedom -the recipe does not refine, so per-parameter flags are honoured. The native -schema has already refused any flag pattern GSAS-II could not honour (a strict -subset of a tie group or of a GSAS-II cell group), so no flag is ever silently -overridden. - -### What the native gsasii path adds around the frozen code - -- `sites.py` compares GSAS-II's `(site symmetry, multiplicity)` of every atom - with core's after phase setup and stops with an error naming an equivalent - position when GSAS-II misreads the site (some 2-fold sites of R32/R-3m/R-3c) - or cannot name it while a coordinate / `Uaniso` is refined. -- `fit_report.py` computes the uniform statistics over the stated window, - reads GSAS-II's stored messages into `gsasii_refinement_message` warnings, - its convergence flag into `engine_details.converged`, counts - `parameters_varied` from GSAS-II (`Nvars`; the Peak List count after - singular drops), and raises on a non-finite calculated pattern. -- The documented defaults (size/strain 10 µm / 0 with a warning; background - none) are pydantic defaults in `schema.py`, so the frozen setters receive - explicit values and GSAS-II's own 1 µm / 1000 / constant-1.0 defaults are - never used. - -### Known GSAS-II quirks - -All on the [engine behaviour page](engine-behavior.md); the ones a developer -meets first: the background-peak width `BkPksig` is a **variance** (the -schema names it `pv_gaussian_sigma_sq`); SH/L is floored at 0.002 in the -profile calculation; `Refine` returns nothing and stores its messages in the -project; `Nvars` counts parameters dropped as singular; `GOF` means two -different things in Rietveld and Peak List fits; limits are applied after the -cycles (why gsasii 0.1.0 exposes no bounds). - -### Output files - -Fixed filenames in every engine (`refined_parameters.csv`, -`_unit_cell_report.csv`, `_peak_list_report.csv`, -`fit_profile.txt`; single peak fitting adds a peak report), written by -`powderline.reports` with one column order and LF line endings so the files -are byte-stable across platforms. gsasii keeps `dummy.gpx`, `dummy.bak0.gpx` -and `dummy.lst`; topas keeps the INP, `.xye` and every file TOPAS wrote; -easydiffraction writes the reports only. Refinement runs add -`output_recipe.json` and `parameter_dispositions.csv`. `powderline.run()` -removes every PowderLine-owned fixed name (`reports.FIXED_OUTPUT_NAMES` / -`FIXED_OUTPUT_PATTERNS`) from the directory before dispatch and never the -engine's files; a failed run leaves the engine's files plus `failure.json` -(the envelope with a `powderline_provenance` block) and no result table — -in gsasii the checks run before the tables are written. - - The `refined_parameters.csv` -columns: `parameter_name` (the engine's identifier), `descriptive_name`, -`phase_name`, `phase_idx`, `atom_name`, `atom_idx`, `value`, `esd`, -`category`. gsasii's ESDs come from GSAS-II's covariance matrix -(`proj.data['Covariance']`); unit-cell ESDs are converted from the -reciprocal-metric tensor with `extractors.calculate_cell_esds_from_A_matrix`. - -## Domain essentials - -**Rietveld refinement** fits a calculated powder pattern to the observed one -by adjusting structural, profile and background parameters. The headline -quality metric, in PowderLine's uniform form (percent, over the fit window's -points with positive weight): - -$$R_{wp} = 100 \sqrt{\frac{\sum_i w_i (y_{obs,i} - y_{calc,i})^2}{\sum_i w_i y_{obs,i}^2}}, \quad -R_{exp} = 100 \sqrt{\frac{N - P}{\sum_i w_i y_{obs,i}^2}}, \quad -\mathrm{GoF} = R_{wp}/R_{exp}$$ - -with N the included points and P the engine's count of varied parameters. -Lower Rwp is a better fit of *that model* to *those points*; across engines -compare lattice parameters, not Rwp (the models differ: background peaks, -asymmetry treatment, sample broadening). - -**Parameter types** (every one a `[value, flag]` or `[value, flag, min, max]` -list): structural (cell, coordinates, occupancies, ADPs: the core phase block, -identical across engines), intensity (phase `scale`), peak shape (sample -size/strain in gsasii and topas; the instrument profile in every engine), -background (Chebyshev; gsasii/topas peaks), instrument (profile terms, -wavelength, zero shift, polarization, asymmetry: the engine's own names and -units). - -**Phase vs histogram parameters**: phase parameters belong to a crystal -structure (cell, atoms, scale, sample broadening); histogram parameters belong -to the measurement (instrument profile, background) and are shared across -phases in a multi-phase refinement. - -**Glossary**: *Recipe* — PowderLine's JSON input. *Gateway* — the per-engine -package behind the registry. *Schema layer / runtime layer* — the engine-free -and engine-importing halves of a gateway. *Tie group* — parameters the -symmetry makes one parameter (cubic a, b, c). *Golden* — a committed engine -output a test compares against. *GPX / LST* — the GSAS-II project and log. -*INP / `.xye`* — TOPAS's input and data files. - -## Development workflow - -### Environments and tasks - -```bash -pixi install -e dev # every engine + docs/test tooling (contributors) -pixi install -e gsasii # GSAS-II only -pixi run kicker --validate-only # default env: engine-free -pixi run -e gsasii kicker # run a gsasii.* recipe -pixi run -e easydiffraction kicker # run an easydiffraction.* recipe -pixi run test # engine-free suites (core + topas_gen), any env -pixi run -e dev test-all # every suite, the pre-merge gauntlet -pixi run -e gsasii pytest tests/test_example_LaB6_regression.py -q # one engine's suite -pixi run docs # Sphinx with -W (zero warnings is the standard) -pixi run -e gsasii gsas-server start|status|stop|restart|logs -pixi run update-code-hash # after a signed edit to a frozen gsasii module -pixi reinstall powderline --all # after editing entry points / [project.scripts] -``` - -The default feature carries only PowderLine's runtime dependencies plus -pytest; `gsasii`, `easydiffraction`, `all` and `dev` are features/environments -in `pixi.toml`. `test` and `update-code-hash` live in the default feature and run anywhere; -tasks only `dev` has (`test-all`, `docs`, `docs-clean`) resolve without `-e`; engine-bound tasks live in their engine's feature, so pixi -reports them as ambiguous without `-e` and prints the form to use. Tasks that -import PowderLine set `PYTHONPATH` through the task-level `env` dict (cross-platform; -[cross-platform guide](cross-platform-guide.md)). - -### Execution modes (gsasii) - -| Mode | Command | Behaviour | -|---|---|---| -| auto (default) | `pixi run -e gsasii kicker input.json` | tries the server, auto-starts it, falls back to in-process | -| server | `... --use-server` | server only; fails if it cannot be started | -| in-process | `... --no-server` | skips the server; GSAS-II imported in this process | - -The server amortizes GSAS-II's import (~2.6 s) over many runs (batch jobs: -0.1–0.3 s per run against 1–2 s in-process). It **caches code**, so restart it -after editing anything under `src/` it executes; since 0.2.0 a forgotten -restart is caught: `/health` reports the PowderLine version and an MD5 of the -source tree the server loaded (`server.source_tree_hash()`), the client -compares both with its own and treats a mismatch as a **stale** server — -`auto` mode says so and runs in-process, `--use-server` fails with -`server_unavailable` naming `gsas-server restart`, and `gsas-server status` -prints `Code: STALE`. - -**No shared filesystem (KI-12):** the client sends the recipe inline and -the server runs each job in a private scratch directory under its state dir; -every output file comes back base64-encoded inside the signed response -(`files`, GSAS-II's `*.bak*.gpx` backups excepted) and the client writes it -into its own `output_dir` (atomic replace, retried on Windows). A server on -another node or in a container therefore behaves like a local one; a client -that cannot write its directory fails with `backend_error`, a malformed or -oversized transfer with `communication_failed` (auto mode then runs -in-process). - -**Server state and security (per user, v0.1.2):** the server binds `127.0.0.1` -on a free ephemeral port and writes its lock, PID, log, and an endpoint file -(port + random secret token) — all mode 0600 — to a private per-user state -directory: `$XDG_RUNTIME_DIR/powderline` (if it is a real directory owned by -you), else `/powderline-` on POSIX (mode 0700; refused if it is a -symlink, owned by someone else, or group/other-accessible, or if the temp -directory would let others replace it), and `%LOCALAPPDATA%\powderline` on -Windows. On POSIX the files are opened relative to a re-verified, no-follow -directory handle. - -The token never travels over the wire: every request carries a fresh nonce, a -timestamp and an HMAC-SHA256 over method, path, nonce, timestamp and body, and -the server rejects anything unsigned, wrongly signed, stale, or replayed — on -every route, including 404/405/422 — before reading the body. Every response -is signed back over the nonce, status and body, and the client discards -unsigned responses. Other users on a shared host therefore can neither submit -jobs to your server nor impersonate it (e.g. by taking over the port of a -crashed server). `gsas-server start` takes a kernel-held lock (`flock` / -`msvcrt.locking` on `server.lock`) for the server's lifetime; the OS releases -it when the process exits, so concurrent auto-starts never spawn duplicates -and leftover PID files can never make a dead server look alive. - -**One refinement at a time:** the server runs `/simulate` requests strictly in -sequence (GSAS-II's module-level state is not thread-safe); concurrent clients -queue (up to the ASGI thread pool's size, 40 by default; `/health` is not -behind the lock). **Port problems are reported, not crashed:** a malformed -`POWDERLINE_SERVER_PORT` is refused up front by `gsas-server start` and by a -client about to auto-start (`ExecutionBackendError`, `server_unavailable`, -naming the variable); a busy port fails the start before GSAS-II is imported, -with the port named, and a client whose auto-started server exits at start -reports the exit and points at the server log. - -**Pin the server port** (only if needed — the default ephemeral port avoids -collisions between users): -```bash -export POWDERLINE_SERVER_PORT=19472 -pixi run -e gsasii gsas-server start -pixi run -e gsasii kicker input.json # the client reads the port from your state directory -``` - -**Disable the server entirely** (shared workstations, locked-down deployments): -```bash -export POWDERLINE_NO_SERVER=1 # also: true / yes / on -``` -With it set, `execution_mode='auto'` (and the CLI default) runs in-process and -never starts a server, `execution_mode='server'` / `--use-server` fail with a -clear error, and `pixi run -e gsasii gsas-server start|restart` refuse. - -**View server logs** (`server.log` in the state directory; `status` prints the path): -```bash -pixi run -e gsasii gsas-server logs -``` - -### Adding a new recipe field (any engine) - -1. **Schema**: add the field to the engine's `schema.py` (unit metadata, - validators, refusals; for a structural quantity, to `schema_core`). -2. **Gateway**: consume it from the validated model in the runtime layer - (gsasii: a non-frozen module, or a signed-off edit for a frozen read). -3. **Fixture + golden**: a native recipe stating it and the engine's output. -4. **Example** if it is a real case: `examples//` with `DESCRIPTION.md` - and, for gsasii, the committed output (regression reference). -5. **Docs**: `scripts/gen_schema_field_tables.py`, SCHEMA_HISTORY, the - input-guidance / engine-behaviour pages as needed, CHANGELOG. - -### Adding a new example - -Examples drive schema evolution; see `examples/CONTRIBUTING_EXAMPLES.md`. -Directory `examples//` with `input.json` (the `gsasii.*` recipe), -`DESCRIPTION.md`, `output/` (committed GSAS-II results as regression -references), and optionally `topas/input.json`, `easydiffraction/input.json` -with a README each. An `easydiffraction/output/` folder holds that engine's -committed reference results (checked against the easydiffraction goldens); -TOPAS references live as goldens under `tests/data/topas/native/golden//` -(they come from a TOPAS machine; the directory is the fixture's `engine_version`). - -### Running tests - -Every test carries exactly one suite marker (`pyproject.toml` -`[tool.pytest.ini_options] markers`; `tests/conftest.py` refuses a collection -where one is missing or doubled), and the pixi tasks select on them: - -```bash -# Engine-free suites (core + topas_gen): any env, the default one included. -# Installs the core guard: an engine import or a tc.exe run fails loudly. -pixi run test - -# Engine suites, each in its env (the slow regressions are part of test-gsasii) -pixi run -e gsasii test-gsasii -pixi run -e easydiffraction test-easydiffraction - -# TOPAS: generation everywhere; the topas_exec tests run tc.exe on the TOPAS -# machine when POWDERLINE_TOPAS_DIR and POWDERLINE_TOPAS_VERSION are set -# (PowerShell: $env:POWDERLINE_TOPAS_DIR='C:\TOPAS6'; $env:POWDERLINE_TOPAS_VERSION='6') -pixi run test-topas - -# Everything, in the dev env -- the pre-merge gauntlet -pixi run test-all - -# Extra pytest arguments pass through -pixi run test -x -k schema -pixi run -e gsasii test-gsasii -m "gsasii and not slow" -``` - -Markers: `core` (imports no engine, runs no engine subprocess -- the schema -layer of every gateway included), `gsasii`, `topas_gen` (engine-free TOPAS -input generation), `topas_exec` (tc.exe), `easydiffraction`, plus the optional -`slow` (a full refinement runs). `POWDERLINE_CORE_GUARD=1` forces the guard on -for a path or `-k` selection; `=0` turns it off. - -**Regression test strategy:** -- Rwp, unit cell parameters and refined values compared within the tolerances - of `regression-tolerance.md`; peak lists by row count -- Detects any change in the engine's refinement result -- Reference outputs committed to git for comparison -- If a test fails: inspect the `.lst` file to determine whether the change is - expected or a bug; a new engine version goes through `engine-versions.md` - - -Suite map: `test_schema_core`, `test_symmetry_core`, -`test_symmetry_characterization` (core rules; all 230 groups against -GSAS-II's site tables), `test_phase_canary` (engine phase models cannot -weaken core), `test_api`, `test_cli`, `test_registry`, `test_compat`, -`test_exceptions`, `test_fitstats`, `test_reports` (core), `test_gsasii_schema`, -`test_gsasii_schema_engine` (GSAS-II's reading of every setting), -`test_gsasii_native_run`, `test_example_LaB6_regression`, `test_esd_extraction`, -`test_per_parameter_flags`, `test_run_refinement_missing_keys`, -`test_constraints_mapping`, `test_code_hash`, the server/client tests -(gsasii), `test_topas_schema`, `test_topas_native`, `test_topas_native_golden` -(12 INPs byte-identical to the TOPAS 6 goldens), -`test_topas_runner` (topas), -`test_easydiffraction_schema`, `test_easydiffraction_native`, -`test_easydiffraction_space_groups` (easydiffraction), -`test_convert_recipe_026` (the migration tool), `test_cross_engine_comparison`, -`test_simulation_builder` (the structure → simulation-recipe builder). - -**Regression strategy:** the gsasii examples' outputs are compared within -the tolerances of [regression-tolerance.md](regression-tolerance.md) -(`tests/subprocess_utils.py`: `refined_params_match`, `unit_cell_match`), -peak lists by row count, GSAS-II's own `.lst` wR as a freeze tripwire and the -uniform `rwp` against a recorded value; TOPAS and easydiffraction against -their goldens (easydiffraction cell rtol 1e-4, values 1e-3). If a regression -test fails, inspect the engine's log and the deliberate-changes policy before -touching a reference. Some gsasii cases need a working GSAS-II refinement -and can fail for environment reasons: compare against the baseline run. - -### Documentation - -`pixi run docs` builds with `-W --keep-going` and `nitpicky`; zero warnings -is the standard. Pages under `docs/_generated/` are written by scripts -(`gen_schema_field_tables.py`: field tables and support matrix; -`cross_engine_comparison.py`: the comparison tables) and pulled in with MyST -`{include}`; regenerate them with the change that affects them. `api.rst` -documents the public API; the schema contract is the generated field tables. - -### Template file detection - -The CLI refuses a recipe whose path contains `template` or whose payload lacks -the blocks its workflow needs (`xrd_data`, `instrument`, `refinement_controls` -and `phases` or `single_peaks`), pointing at the examples. It is a CLI-only -guard: `powderline.run()` trusts its caller. - -## Key gotchas - -- **Restart the GSAS-II server** after code changes; a stale server is the - usual cause of "my fix has no effect". -- **`pixi reinstall powderline --all`** after touching entry points or - `[project.scripts]`, or when the registry reports no gateways (KI-13). -- **Frozen modules** change only through signed-off, recorded edits; `test_code_hash` guards - them; `pixi run update-code-hash` after a signed edit. -- **Generated docs** go stale silently unless regenerated - (`gen_schema_field_tables.py --check`). -- **Parameter correlation warnings** from GSAS-II (`gsasii_refinement_message`: - "parameters 0 and 1 are 99.8% correlated") are normal for - over-parameterized models (background terms, U/V/W, size vs strain); - they matter when a fit does not converge or values are unphysical. -- **Chebyshev coefficients** are indexed from the constant term; their basis - spans the fit window's first and last data points in every engine. -- **Single peak fitting**: `gsasii.spf` widths are σ² (centidegree²) and γ - (centidegree), each ≥ 0.001; with `use_instrument_profile: true` the - widths come from U..Z and the peaks' own width flags must be `false` - (GSAS-II would ignore them silently); `topas.spf` peaks are `xo_Is` with - the instrument peak type convolved with their own `gauss_fwhm` / `lor_fwhm`. - gsasii writes `single_peaks_report.txt` (position, intensity, σ, γ, FWHMs, - integral breadths, convergence) and `peak_convergence_diagnostics.txt` only - when a peak did not converge. -- **Error handling**: validation errors are pydantic's, listed together; - PowderLine's own errors derive from `PowderLineError`; the CLI exits 1 on - any failure. - -## The simulation recipe builder - -`powderline.gateways.gsasii.simulation_builder.SimulationRecipeBuilder` turns a -crystal-structure dict (`space_group`, `unit_cell`, `atoms`, `formula`) into a -`gsasii.rietveld` 0.1.0 simulation recipe: the conventional cell reduced to -symmetry-unique sites with an orbit self-check, two-origin groups moved to -origin choice 2, the canonical space-group name, every default stated, SH/L -0.002, instrument defaults from `.powderline_config.yaml`'s -`simulation_defaults`. The Materials Project fetch and `.chi` export that -drove it (`mp-simulate`) left PowderLine in 0.2.0; RandomXRD carries -its own copy. The recipe's `metadata.materials_project` block (material id, -formula, the space group as stated) keeps its name: it is what the tests and -RandomXRD's copy read, and a source-neutral key is a shape decision for the -maintainer. - -## Future roadmap - -- A later minor gsasii - schema version with bounds (clamp-freeze-refit); TOPAS 7 after a - compatibility run on a TOPAS 7 machine; `unsafe_mode` for deliberately - relaxed checks; a recipe builder / CIF importer upstream of the schemas; - a structure-source plugin (Materials Project, CIF) upstream of the builder. - -## Contributing - -Key principles for future work: -- Add features when real examples need them (not speculatively) -- Every rule in the schema; every engine quirk refused or warned about, and - recorded in the engine-behaviour register -- Behaviour changes are deliberate, recorded and reviewed; references move - only with an equivalence record -- Frozen modules only through signed-off, recorded edits; "ask, don't assume" on any open - decision ([policies](policies.md)) -- Add regression tests, fixtures and goldens for new capabilities - -Questions? Open an issue or discussion on GitHub. diff --git a/docs/SCHEMA_HISTORY.md b/docs/SCHEMA_HISTORY.md deleted file mode 100644 index 83d9b55..0000000 --- a/docs/SCHEMA_HISTORY.md +++ /dev/null @@ -1,812 +0,0 @@ -# Schema Evolution History - -PowderLine recipes are validated against a **core schema** (the shared top level -and structural models, owned by PowderLine) plus one **engine schema** per -gateway (`gsasii`, `topas`, `easydiffraction`), each in its engine's native -conventions. Every schema has its own version. A recipe states both -(`core_schema_version`, `engine_schema_version`), and each PowderLine release -**declares** which versions it accepts. Compatibility is never inferred from the -version numbers (`powderline.support_matrix()` lists the declarations). - -> **PowderLine 0.2.0** reads these schemas only. The unified 0.26.0 schema of -> PowderLine 0.1.1 is documented in the archive at the bottom of this page and -> is no longer accepted; `scripts/convert_recipe_026.py` converts a 0.26.0 -> GSAS-II recipe (see the [migration guide](migration-0.1-to-0.2.md)). The -> field tables generated from these schemas are on the [fields page](fields.md). - -## Version numbering: a new line starting at 0.1.0 - -PowderLine 0.2.0 replaces the single recipe schema of PowderLine 0.1.x — -versions 0.1 to **0.26.0**, `schema_name` `GSASII_Rietveld` / `GSASII_SPF` — -with a **core schema** and **one schema per engine** (`gsasii`, `topas`, -`easydiffraction`). Each is versioned separately, and each starts at -**0.1.0** in PowderLine 0.2.0 (`"core_schema_version": "0.1.0"`, -`"engine_schema_version": "0.1.0"`). - -- The old line **ends at 0.26.0**. It is told apart from the new schemas by - `schema_name` (`GSASII_Rietveld` versus `gsasii.rietveld`) and by the split - into `core_schema_version` + `engine_schema_version` (instead of one - `schema_version`), **not by the number**: a new schema at 0.1.0 is not - older than 0.26.0, it is a different schema. -- The new schemas are 0.x because PowderLine is pre-production and they will - still evolve; the bump rule for 0.x versions is in - [policies](policies.md#versioning-and-compatibility). -- `scripts/convert_recipe_026.py` migrates a 0.26.0 GSAS-II recipe to - `gsasii.*` 0.1.0; there is no converter to `topas.*` or `easydiffraction.*` - (see the [migration guide](migration-0.1-to-0.2.md)). - ---- - -## Core schema - -### 0.1.0 — introduced in PowderLine 0.2.0 - -Defined in `src/powderline/schema_core.py`; every engine schema builds on it. - -> **Pre-release revision (not a new version).** Before its first -> release, core 0.1.0 replaced its structure-only phase model (`PhaseStructure` -> with plain values, plus `null`-valued "structure" parameters) with the -> **phase block** described below: one block per phase, every structural -> quantity a refinable parameter, flags and bounds checked against the symmetry, -> and `space_group` in gemmi's canonical spelling. No recipe was ever written -> against the earlier draft. - -- **Recipe frame** (`CoreRecipe`): `schema_name` (`"."`), - `core_schema_version`, `engine_schema_version`, `metadata`, `payload` (typed - by the engine schema). Unknown keys are errors at every level. -- **`metadata`**: free-form and never interpreted; must be JSON-serializable and - at most **1 MiB** as UTF-8 JSON. The cap is the constant - `schema_core.METADATA_MAX_BYTES`; change it there and record the change here. -- **Parameters**: `[value, refine_flag]` (no bounds) or - `[value, refine_flag, min, max]` (bounds the engine honors; `min`/`max` may - be `null` = open). A bounded list where bounds aren't supported is an error. - `value` is always a number and `refine_flag` a JSON boolean; neither is ever - `null`. -- **Data and ranges**: `xrd_data` (2θ in degrees, weights 1/σ², validated as in - 0.26.0); `fit_range` `[min, max]`, with `max > min`, inside the data's 2θ range, holding at least two points with a positive weight. -- **Background**: Chebyshev (`num_coefficients`, `coefficients`, `refine_flag`). - Coefficient semantics are documented per engine. -- **Units** are fixed per field in the schema, never written in a recipe. -- **Numbers** are JSON numbers: a quoted number (`"0.25"`) or a boolean is an - error, never converted. Integer fields (`Multiplicity`, `num_coefficients`) - take a whole number (`4` or `4.0`), not `4.5`. -- **Accepted core versions** (this release): `==0.1.0`. Compatibility is - declared, never inferred ([support matrix](support-matrix.md)). - -- **Phase block** (`Phase`): one block per phase, keyed by the phase name in - the engine payload's `phases` (no `phase_name` field). Core owns - `space_group`, `unit_cell` and `atoms`; each engine schema adds its own phase - fields next to them (e.g. gsasii `scale`, `peak_broadening`) and may add - its own checks, but never redefine, skip or rewrite core's fields and - validation (checked when the engine schema is defined, and by a test). Every structural quantity is a parameter (`[value, flag]`, - or `[value, flag, min, max]` in engines with bounds): cell `a`–`gamma`; atom - `x`, `y`, `z`, `occupancy`, `Uiso` or `U11`…`U23`. `element`, `ADP`, - `Multiplicity` and `space_group` are plain values. Nothing is `null` and - nothing has a default: `occupancy` must be stated; `Multiplicity` (derived, - so optional) is left out when not stated, never `null`. - **Names:** atom labels (the `atoms` keys) and phase names (the engine - payload's `phases` keys) start with a letter and contain only ASCII letters, - digits and `_` (`O1`, `LaB6_a`); phase names, and the atom labels of a - phase, must also differ by more than case. This is the form every engine can use as-is: GSAS-II renames a phase - with surrounding spaces or non-ASCII characters (and its settings were then - skipped), easydiffraction accepts no other atom label, TOPAS writes the names - into its input file, phase names become report file names (which ignore case - on Windows and macOS), and problems are reported at `.`-joined paths - (`atoms.O1.Uiso`). So `"2H-MoS2"` is written e.g. `MoS2_2H`. - Structural interpretation is checked once, in core, so every engine gets the - same structure. **The recipe is the record of the refinement intent**, and - validation never changes what it means. Values fixed or tied by symmetry - follow one rule (see *Symmetry-determined values* below): the first member - of each tie group is stated freely and kept as written; every other value is - derived from the symmetry and must be stated, either exactly (a constant - with a decimal form, such as `0.5` or `0`) or to the precision that states it - beyond doubt (a third, `0.333333`; a value derived from another stated - value). The validated model, which every engine receives, holds the derived - values. PowderLine never rewrites a recipe file; only a Python dump of a - validated model shows a derived value's full spelling - (`0.3333333333333333`), which means the same and validates the same. All problems - are reported together, each at its own field - (`unit_cell`, `atoms.