diff --git a/pkgs/applications/editors/emacs/elisp-packages/manual-packages/treesit-grammars/package.nix b/pkgs/applications/editors/emacs/elisp-packages/manual-packages/treesit-grammars/package.nix index aa1931869d6b4..ed8e4eaa35212 100644 --- a/pkgs/applications/editors/emacs/elisp-packages/manual-packages/treesit-grammars/package.nix +++ b/pkgs/applications/editors/emacs/elisp-packages/manual-packages/treesit-grammars/package.nix @@ -15,9 +15,9 @@ let # Usage: # treesit-grammars.with-grammars (p: [ p.tree-sitter-bash p.tree-sitter-c ... ]) - with-grammars = fn: grammarPackage (fn pkgs.tree-sitter.builtGrammars); + with-grammars = fn: grammarPackage (fn pkgs.tree-sitter-grammars.derivations); - with-all-grammars = grammarPackage pkgs.tree-sitter.allGrammars; + with-all-grammars = grammarPackage pkgs.tree-sitter-grammars.allGrammars; in { inherit with-grammars with-all-grammars; diff --git a/pkgs/applications/editors/vim/plugins/nvim-treesitter/overrides.nix b/pkgs/applications/editors/vim/plugins/nvim-treesitter/overrides.nix index 84051feecadc8..9eb050654f911 100644 --- a/pkgs/applications/editors/vim/plugins/nvim-treesitter/overrides.nix +++ b/pkgs/applications/editors/vim/plugins/nvim-treesitter/overrides.nix @@ -4,6 +4,7 @@ symlinkJoin, vimUtils, tree-sitter, + tree-sitter-grammars, neovim, neovimUtils, runCommand, @@ -127,7 +128,7 @@ let withPlugins = f: let - selectedGrammars = f (tree-sitter.builtGrammars // builtGrammars); + selectedGrammars = f (tree-sitter-grammars.derivations // builtGrammars); grammarPlugins = map grammarToPlugin selectedGrammars; diff --git a/pkgs/by-name/di/diffsitter/package.nix b/pkgs/by-name/di/diffsitter/package.nix index 5397cd711ed12..30b802e3149fb 100644 --- a/pkgs/by-name/di/diffsitter/package.nix +++ b/pkgs/by-name/di/diffsitter/package.nix @@ -4,32 +4,19 @@ linkFarm, makeWrapper, rustPlatform, - tree-sitter, + tree-sitter-grammars, gitUpdater, versionCheckHook, }: let # based on https://github.com/NixOS/nixpkgs/blob/aa07b78b9606daf1145a37f6299c6066939df075/pkgs/development/tools/parsing/tree-sitter/default.nix#L85-L104 - withPlugins = - grammarFn: - let - grammars = grammarFn tree-sitter.builtGrammars; - in - linkFarm "grammars" ( - map ( - drv: - let - name = lib.strings.getName drv; - in - { - name = "lib" + (lib.strings.removeSuffix "-grammar" name) + ".so"; - path = "${drv}/parser"; - } - ) grammars - ); + grammarToAttrSet = drv: { + name = "lib" + (lib.strings.removeSuffix "-grammar" (lib.strings.getName drv)) + ".so"; + path = "${drv}/parser"; + }; - libPath = withPlugins (_: tree-sitter.allGrammars); + libPath = linkFarm "grammars" (map grammarToAttrSet tree-sitter-grammars.allGrammars); in rustPlatform.buildRustPackage rec { pname = "diffsitter"; diff --git a/pkgs/by-name/he/helix/package.nix b/pkgs/by-name/he/helix/package.nix index fe7aca3037704..e301693489f6b 100644 --- a/pkgs/by-name/he/helix/package.nix +++ b/pkgs/by-name/he/helix/package.nix @@ -6,7 +6,7 @@ helix-unwrapped, removeReferencesTo, pkgs, - tree-sitter, + tree-sitter-grammars, lockedGrammars ? lib.importJSON ./grammars.json, grammarsOverlay ? ( final: prev: { @@ -66,13 +66,9 @@ let } ) prev; - tree-sitter-grammars = + helixTreeSitterGrammars = lib.filterAttrs (drvName: _: lib.hasAttr (lib.removePrefix "tree-sitter-" drvName) lockedGrammars) - ( - tree-sitter.grammarsScope.overrideScope ( - lib.composeExtensions lockedVersionsOverlay grammarsOverlay - ) - ); + (tree-sitter-grammars.overrideScope (lib.composeExtensions lockedVersionsOverlay grammarsOverlay)); # Dynamic libraries for the grammars always use the `.so` extension, also on Darwin (should use `.dylib`) # See here: https://github.com/helix-editor/helix/pull/14982 @@ -82,7 +78,7 @@ let lib.concatMapAttrsStringSep "\n" (_: grammar: '' install -D ${grammar}/parser $out/${grammar.language}.so ${lib.getExe removeReferencesTo} -t ${grammar} $out/${grammar.language}.so - '') (lib.filterAttrs (_: lib.isDerivation) tree-sitter-grammars) + '') helixTreeSitterGrammars ); lockedGrammarsCount = lib.length (lib.attrNames lockedGrammars); @@ -113,7 +109,7 @@ symlinkJoin { passthru = { updateScript = ./update.sh; runtime = runtimeDir; - inherit tree-sitter-grammars; + tree-sitter-grammars = helixTreeSitterGrammars; }; meta = { diff --git a/pkgs/by-name/tr/tree-sitter/grammars/README.md b/pkgs/by-name/tr/tree-sitter/grammars/README.md index 9445f68494cb0..c829bf6aaa2ec 100644 --- a/pkgs/by-name/tr/tree-sitter/grammars/README.md +++ b/pkgs/by-name/tr/tree-sitter/grammars/README.md @@ -109,6 +109,59 @@ This includes build-related flags and metadata. } ``` +## Overriding the Grammar Set + +Use `pkgs.tree-sitter-grammars.overrideScope` when adding a grammar or replacing a grammar that another package should consume. +`pkgs.tree-sitter-grammars` is the scoped package set used for grammar overrides and scoped helpers such as `derivations`, `allGrammars`, and `withPlugins`. + +```nix +let + grammars = pkgs.tree-sitter-grammars.overrideScope ( + final: prev: { + tree-sitter-foolang = pkgs.tree-sitter.buildGrammar { + language = "foolang"; + version = "0.42.0"; + src = pkgs.fetchFromGitHub { + owner = "example"; + repo = "tree-sitter-foolang"; + rev = "v0.42.0"; + hash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + }; + }; + + tree-sitter-rust = prev.tree-sitter-rust.overrideAttrs (_: { + version = "custom"; + src = pkgs.fetchFromGitHub { + owner = "example"; + repo = "tree-sitter-rust"; + rev = "custom"; + hash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + }; + }); + } + ); +in +grammars.withPlugins (p: [ + p.tree-sitter-foolang + p.tree-sitter-rust +]) +``` + +The scoped `withPlugins` helper receives derivations from the same overridden scope, so added or replaced grammars are visible. + +The set also carries package-set helpers (`callPackage`, `newScope`, `overrideScope`, …) alongside the grammars, so do not iterate it directly. +Use one of its grammar-only views instead; each reflects any `overrideScope`: + +- `pkgs.tree-sitter-grammars.derivations` — attrset of every grammar derivation, including grammars marked broken. +- `pkgs.tree-sitter-grammars.allGrammars` — list of the non-broken grammar derivations. +- `pkgs.tree-sitter-grammars.withPlugins` — build a grammar link farm. + +```nix +builtins.attrValues pkgs.tree-sitter-grammars.derivations +``` + +`pkgs.tree-sitter.builtGrammars` remains the plain attribute set generated directly from [grammar-sources.nix](grammar-sources.nix); use it when you specifically want the stock grammars without any scope overrides. + ## Building WebAssembly Parsers `buildGrammar` builds a native `$out/parser`. diff --git a/pkgs/by-name/tr/tree-sitter/package.nix b/pkgs/by-name/tr/tree-sitter/package.nix index cc4522b35f217..7ed95889f5d28 100644 --- a/pkgs/by-name/tr/tree-sitter/package.nix +++ b/pkgs/by-name/tr/tree-sitter/package.nix @@ -75,26 +75,12 @@ let */ builtGrammars = lib.mapAttrs (_: lib.makeOverridable buildGrammar) grammars; - /** - # Extensible package set for tree-sitter grammars. - # Provides .override and .extend for customization. - # Note: Use builtGrammars (not this) when iterating over grammars, - # as this includes package set functions alongside derivations - */ - grammarsScope = lib.makeScope newScope (self: builtGrammars); - - # Usage: - # pkgs.tree-sitter.withPlugins (p: [ p.tree-sitter-c p.tree-sitter-java ... ]) - # - # or for all grammars: - # pkgs.tree-sitter.withPlugins (_: pkgs.tree-sitter.allGrammars) - # which is equivalent to - # pkgs.tree-sitter.withPlugins (p: builtins.attrValues p) - withPlugins = - grammarFn: - let - grammars = grammarFn builtGrammars; - in + grammarDerivationsFrom = lib.filterAttrs ( + name: value: lib.hasPrefix "tree-sitter-" name && lib.isDerivation value + ); + + mkGrammarLinkFarm = + grammars: linkFarm "grammars" ( map ( drv: @@ -112,7 +98,32 @@ let ) grammars ); - allGrammars = lib.filter (p: !(p.meta.broken or false)) (lib.attrValues builtGrammars); + /** + Extensible package set of compiled tree-sitter grammars. + + Exposed as `pkgs.tree-sitter-grammars` and `pkgs.tree-sitter.grammarsScope`. + Customize with `.overrideScope`; overrides propagate to every consumer that + reads the scope, including the grammar-only views below (which the + `pkgs.tree-sitter` passthru re-exports so there is a single source of truth): + + `.derivations` attrset of every grammar derivation + `.allGrammars` list of non-broken grammar derivations + `.withPlugins` build a grammar link farm + + The scope also carries package-set helpers (`callPackage`, `overrideScope`, + …) alongside the grammars, so prefer one of the views above when iterating. + */ + grammarsScope = lib.makeScope newScope ( + self: + builtGrammars + // { + derivations = grammarDerivationsFrom self; + allGrammars = lib.filter (p: !(p.meta.broken or false)) ( + lib.attrValues (grammarDerivationsFrom self) + ); + withPlugins = grammarFn: mkGrammarLinkFarm (grammarFn (grammarDerivationsFrom self)); + } + ); isWasi = stdenv.hostPlatform.isWasi; @@ -237,14 +248,15 @@ rustPlatform.buildRustPackage (finalAttrs: { passthru = { inherit - grammars buildGrammar builtGrammars + grammars grammarsScope - withPlugins - allGrammars ; + # Keep legacy `pkgs.tree-sitter` views wired to the overridable scope. + inherit (grammarsScope) allGrammars withPlugins; + updateScript = nix-update-script { }; tests = { diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 86aaa22569760..c945dd809bbeb 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -5450,7 +5450,7 @@ with pkgs; tflint-plugins = recurseIntoAttrs (callPackage ../development/tools/analysis/tflint-plugins { }); - tree-sitter-grammars = recurseIntoAttrs tree-sitter.builtGrammars; + tree-sitter-grammars = recurseIntoAttrs tree-sitter.grammarsScope; uhdMinimal = uhd.override { enableUtils = false; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 81b1bd4dac5a2..ee00178ee9c79 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -20302,7 +20302,7 @@ self: super: with self; { "tree-sitter-sshclientconfig" "tree-sitter-templ" ]) - ) pkgs.tree-sitter.builtGrammars + ) pkgs.tree-sitter-grammars.derivations ) );