New documentation system

.github/workflows/backend.yml

@@ -79,6 +79,7 @@ jobs:
       - run: npm install
         working-directory: pontoon
       - run: mkdir -p translate/dist translate/public
+      - run: zensical build -f documentation/zensical.toml
       - run: python manage.py collectstatic
 
       # Check if there are missing migrations

.gitignore

@@ -21,6 +21,7 @@ venv/
 /docker/config/server.env
 /docs/_build
 /docs/_gh-pages
+/documentation/site/
 /env/
 /media/
 /pontoon/coverage.xml

.prettierignore

@@ -2,6 +2,7 @@
 translate/dist/
 coverage/
 docs/_build/
+documentation/
 package-lock.json
 specs/
 .venv/

Makefile

@@ -9,7 +9,7 @@ SITE_URL ?= http://localhost:8000
 USER_ID?=1000
 GROUP_ID?=1000
 
-.PHONY: build build-translate build-server server-env setup run clean shell ci test test-translate test-server vitest pytest format lint types eslint prettier check-prettier ruff check-ruff dropdb dumpdb loaddb sync-projects requirements
+.PHONY: build build-translate build-server server-env setup run clean shell ci test test-translate test-server vitest pytest format lint types eslint prettier check-prettier ruff check-ruff dropdb dumpdb loaddb sync-projects requirements docs
 
 help:
 	@echo "Welcome to Pontoon!\n"
@@ -39,7 +39,8 @@ help:
 	@echo "  dumpdb           Create a postgres database dump with timestamp used as file name"
 	@echo "  loaddb           Load a database dump into postgres, file name in DB_DUMP_FILE"
 	@echo "  sync-projects    Runs the synchronization task on all projects"
-	@echo "  requirements     Compiles all requirements files with uv pip compile\n"
+	@echo "  requirements     Compiles all requirements files with uv pip compile"
+	@echo "  docs             Builds the documentation site\n"
 
 translate/dist:
 	make build-translate
@@ -159,3 +160,7 @@ requirements:
 	# Pass --upgrade to upgrade all dependencies
 	# The arguments are passed through to `uv pip compile`
 	"${DC}" run --rm server //app/docker/compile_requirements.sh ${opts}
+
+docs:
+	"${DC}" run --rm server zensical build -f //app/documentation/zensical.toml
+	"${DC}" run --rm server .//manage.py collectstatic --noinput

bin/post_compile

@@ -22,6 +22,9 @@ else
   python manage.py migrate --noinput
 fi
 
+echo "Building documentation..."
+zensical build -f documentation/zensical.toml
+
 echo "Collecting static files..."
 ./manage.py collectstatic --noinput
 

docker-compose.yml

@@ -15,6 +15,7 @@ services:
     ports:
       - '8000:8000'
     volumes:
+      - ./documentation:/app/documentation
       - ./pontoon:/app/pontoon
       - ./requirements:/app/requirements
       - ./translate:/app/translate

docker/Dockerfile

@@ -51,6 +51,9 @@ RUN npm install
 COPY --chown=pontoon:pontoon ./docker/config/server.env .env
 COPY --chown=pontoon:pontoon . /app/
 
+# Build documentation (generates static files)
+RUN zensical build -f documentation/zensical.toml
+
 RUN python manage.py collectstatic
 
 STOPSIGNAL SIGINT

docker/Dockerfile-mozcloud

@@ -54,7 +54,7 @@
 # SECRET_KEY and DATABASE_URL are set only to prevent collectstatic from
 # failing—their actual values don’t matter. collectstatic loads settings/base.py,
 # which requires SECRET_KEY and DATABASE_URL to be set.
 ARG SECRET_KEY=foo
 ARG DATABASE_URL=postgres://pontoon:asdf@postgresql/pontoon
 
 ARG GIT_REVISION=unknown
@@ -92,6 +92,9 @@
 # Copy application code
 COPY . /app/
 
+# Build documentation (generates static files)
+RUN zensical build -f documentation/zensical.toml
+
 # Set up environment variables for collectstatic
 ENV YUGLIFY_BINARY=/app/node_modules/.bin/yuglify
 ENV TERSER_BINARY=/app/node_modules/.bin/terser

documentation/README.md

@@ -0,0 +1,11 @@
+# Pontoon Documentation
+
+This folder contains the unified Pontoon documentation site, built with [Zensical](https://zensical.org/).
+
+It consolidates Pontoon documentation from three previously separate sites:
+
+| Audience | Previous location |
+|---|---|
+| Developers | [docs/](https://mozilla-pontoon.readthedocs.io/en/latest/) |
+| Localizers | [mozilla-l10n/localizer-documentation](https://mozilla-l10n.github.io/localizer-documentation/) |
+| Admins | [mozilla-l10n/documentation](https://mozilla-l10n.github.io/documentation/) |

documentation/docs/admin/adding-new-db-project.md

@@ -0,0 +1,46 @@
+# Adding a New DB Project
+
+DB projects are things like newsletters, marketing campaigns, surveys… They do not have a repository, and data is instead stored only in Pontoon’s database.
+
+The process to create a DB project is very similar to that of a regular one.
+
+## Create the project in Pontoon STAGE instance
+
+First you want to test that everything works using Pontoon staging server.
+
+Access Pontoon’s [admin console](https://pontoon.allizom.org/admin/) on the **stage server** and click **ADD NEW PROJECT**.
+
+* Name: name of the project (it will be displayed in Pontoon’s project selector).
+* Slug: used in URLs, will be generated automatically based on the repository’s name.
+* Locales:
+
+  * Select at least one locale. To make things faster it’s possible to copy supported locales from an existing project.
+  * You can uncheck the `Locales can opt-in` checkbox to prevent localizers from requesting this specific project.
+
+* Data Source: select `Database` in the list of options. This will hide the *Repositories* section and show a *Strings* section instead.
+* Strings: you can enter the initial set of strings here. Strings must be separated by new lines. If you leave this empty, you’ll be able to enter strings as a batch again after creating the project. Strings must be in `en-US`, and they will become the entities on that project. A resource named `database` will automatically be created.
+* For every other option, please refer to the [new project documentation](adding-new-project.md).
+
+Click **SAVE PROJECT** at the bottom of the page, and you’re done!
+
+## Create the project in Pontoon PROD instance
+
+At this point you need to repeat the same steps on the **production server**.
+
+Access Pontoon’s [admin console](https://pontoon.mozilla.org/admin/), add the same information you used on the staging server and make sure to select all supported locales for this project.
+
+The new project will immediately appear in the [public list of Projects](https://pontoon.mozilla.org/projects/).
+
+## Managing strings
+
+After you have created your project, you will be able to manage its source strings. On the admin project page (that you can find from the [admin console](https://pontoon.mozilla.org/admin/)), under the *Strings* section, you will find two links. The one called **MANAGE STRINGS** will lead you to a page where all strings are listed. There you can edit the string content, edit the string comment, add new strings (use the **NEW STRING** button) or remove existing strings (use the trashbin icon under the comment input). Once you’re done editing, click the **SAVE STRINGS** button to save your changes.
+
+## Downloading translations
+
+On the project’s admin page and on the Manage Strings page, you’ll find a **DOWNLOAD STRINGS** link. Clicking it will download a CSV file that contains all the translations in all enabled locales. The file format looks as follow:
+
+```CSV
+Source, fr, de
+Hello, Salut, Hallo
+World, Monde, Welt
+```

documentation/docs/admin/adding-new-locale.md

@@ -0,0 +1,115 @@
+# Adding a New Locale
+
+## Verify if the locale is already available
+
+Access Django’s admin interface at `https://pontoon.mozilla.org/a/` (note that this is not the usual admin interface), then click `Locales`. In the next page search for the locale you want to add (safer to search for the locale code).
+
+## Add the new locale
+
+If the locale you need is not available, click the **ADD LOCALE+** button in the top right corner. For this example, let’s consider that the task is to add Amharic (am).
+
+You will need to complete the following fields in the next page.
+
+### Code
+
+It’s the locale code, in this case `am`.
+
+### Google Translate code
+
+Google Translate maintains a list of supported locales in its own format. Choose one that matches the locale from [a list of supported locales](https://translate.google.com/intl/en/about/languages/) or leave it blank to disable support for Google Translate for this locale.
+
+### Google AutoML model
+
+Set your [Google Cloud AutoML Translation](https://cloud.google.com/translate/) model ID to use the custom translation engine with a trained model. You can find the model ID in [Google Cloud management console](https://console.cloud.google.com/translation/models/list). If the value is not set, Google’s generic machine translation technology is used instead.
+
+### MS translator code
+
+Microsoft Translator maintains a list of supported locales in its own format. Choose one that matches the locale from [a list of supported locales](https://docs.microsoft.com/azure/cognitive-services/Translator/language-support) or leave it blank to disable support for Microsoft Translator for this locale.
+
+### MS terminology code
+
+Microsoft Terminology maintains a list of supported locales in its own format. Choose one that matches the locale from a list provided below the field or leave it blank to disable support for Microsoft Terminology for this locale.
+
+### Name
+
+It’s the language name, in this case `Amharic`.
+
+### Plural rule
+
+It’s the GetText plural rule. This [document](http://docs.translatehouse.org/projects/localization-guide/en/latest/l10n/pluralforms.html) has plural rules for several languages. For example, for Amharic it would be:
+
+```
+nplurals=2; plural=(n > 1);
+```
+
+As explained in the note below the field, you only need to put the last part in the field:
+
+```
+(n > 1)
+```
+
+### CLDR Plurals
+
+You need to find the locale on [CLDR](https://unicode-org.github.io/cldr-staging/charts/latest/supplemental/language_plural_rules.html). For Amharic, there are only two **cardinal** plural forms listed: one, other.
+
+The mapping is:
+
+```
+0 -> zero
+1 -> one
+2 -> two
+3 -> few
+4 -> many
+5 -> other
+```
+
+For Amharic you will need to set the field as `1,5`, separating the available forms with a comma (no spaces).
+
+Irish (ga-IE), for example, has all of them except for 0, so it will be `1,2,3,4,5`.
+
+### Script
+
+The script used by this locale. Find it in [CLDR Languages and Scripts](http://www.unicode.org/cldr/charts/latest/supplemental/languages_and_scripts.html).
+
+### Direction
+
+Writing direction of the script. Set to `right-to-left` if `rtl` value for the locale script is set to `YES` in [CLDR scriptMetadata.json](https://github.com/unicode-cldr/cldr-core/blob/master/scriptMetadata.json).
+
+### Population
+
+This represents the number of native speakers. There are two ways to get this information from CLDR.
+
+#### Using a script
+
+Python needs to be available on the method to use this system: save [this file](https://raw.githubusercontent.com/mozilla-l10n/documentation/main/scripts/cldr_population.py) on your computer as `cldr_population.py` and run it as `python cldr_population.py LOCALE_CODE`.
+
+For example, this is the output of the script when checking data for `sl`:
+
+```
+$ python scripts/cldr_population.py sl
+
+Adding HU: 4937 (0.05% of 9874780)
+Adding IT: 105412 (0.17% of 62007500)
+Adding SI: 1720886 (87% of 1978030)
+Adding AT: 32233 (0.37% of 8711770)
+--------
+Territories: AT, HU, IT, SI
+Population: 1863468
+--------
+```
+
+#### Manual process
+
+Find the locale code in [CLDR territoryInfo.json](https://github.com/unicode-cldr/cldr-core/blob/master/supplemental/territoryInfo.json) and multiply its `_populationPercent` with the territory `_population`. Repeat if multiple occurrences of locale code exist and sum products.
+
+At this point click **SAVE** in the bottom right corner to save and create the new locale. The "Terminology" project should appear automatically under the new team’s page — in this case at https://pontoon.mozilla.org/am/
+
+### Access key localization
+
+In software, access keys allow users to activate a menu command using their keyboard, and they’re displayed as an underlined character in the menu label. Some locales, especially those using a script different than Latin (Chinese, Japanese, etc.), don’t localize access keys, and use the same character as English. In this case, the access key will be appended to the label as an underlined character between parentheses.
+
+Pontoon can recognize an access key attribute in Fluent strings, and provide translators with a dynamic UI to select one of the characters available in the localized label.
+
+When the checkbox is selected (default), Pontoon will leave the access key field empty, and show the dynamic UI when translating. For pretranslation, it will use the first character from the pretranslated label.
+
+When the checkbox is deselected, Pontoon will automatically use the same access key as the source string, for both manual translation and pretranslation.