diff --git a/otterwiki/backlinks.py b/otterwiki/backlinks.py new file mode 100644 index 00000000..fd04b48c --- /dev/null +++ b/otterwiki/backlinks.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 + +""" +When a page is renamed this module rewrites all links +in the wiki that point to the old page so that they point +to the new page name instead. +""" + +import re + +from otterwiki.server import app, storage +from urllib.parse import quote, unquote + +"""Rewrite WikiLinks and images pointing to a renamed page.""" + +# Matches a single WikiLink and captures its inner content. +# WikiLink inner content never contains a closing bracket. +WIKILINK_RE = re.compile(r"\[\[([^\]]+)\]\]") + +# Matches nested images inside links +IMAGE_LINK_WRAPPER_RE = re.compile(r"\[(!\[[^\]]*\]\([^)]+\))\]\((/[^)]+)\)") + +# Matches a Markdown image link +IMAGE_LINK_RE = re.compile(r"!\[([^\]]*)\]\((/[^)]+)\)") + +# Matches Markdown links +# Example: [text](/Parent/ChildPage) +MARKDOWN_LINK_RE = re.compile(r"\[([^\]]+)\]\((/[^)]+)\)") + + +def _is_linktitle_style(): + """True if the link is the *left* side of ``[[left|right]]``.""" + style = ( + app.config.get("WIKILINK_STYLE", "").upper().replace("_", "").strip() + ) + return style in ("LINKTITLE", "PAGENAMETITLE") + + +def _split_link_title(inner, linktitle_style): + """Split a WikiLink's inner content into (link_part, title_part). + + ``title_part`` is None when the link carries no explicit title, in + which case the link doubles as the displayed text. + """ + if "|" in inner: + left, right = inner.split("|", 1) + if linktitle_style: + return left, right + return right, left + # no pipe: the whole content is both link and displayed text + return inner, None + + +def _rewrite_inner(inner, old_key, new_pagepath, linktitle_style): + """Return the rewritten inner content, or None if it doesn't match.""" + from otterwiki.helper import get_filename + + link_part, title_part = _split_link_title(inner, linktitle_style) + + # preserve a leading slash (absolute links) and any #anchor + leading_slash = link_part.startswith("/") + page, hsep, anchor = link_part.strip().lstrip("/").partition("#") + page = unquote(page.strip()) + if not page: + # pure anchor link like [[#section]] - nothing to do + return None + + # resolve the link target to its on-disk filename and compare. + if get_filename(page) != old_key: + return None + + new_link = ("/" if leading_slash else "") + new_pagepath + if hsep: + new_link += "#" + anchor + + if title_part is None: + return new_link + if linktitle_style: + return new_link + "|" + title_part + return title_part + "|" + new_link + + +def _rewrite_attachment(match, old_pagepath, new_pagepath): + """Rewrite Markdown image links pointing to attachments on a renamed page.""" + + alt = match.group(1) + url = match.group(2) + + new_url = _rewrite_url( + url, + old_pagepath, + new_pagepath, + ) + + return f"![{alt}]({new_url})" + + +def _rewrite_markdown_link(match, new_pagepath, old_pagepath): + """Rewrite Markdown links pointing to a renamed page.""" + + text = match.group(1) + url = match.group(2) + + new_url = _rewrite_url( + url, + old_pagepath, + new_pagepath, + ) + + return f"[{text}]({new_url})" + + +def _rewrite_url(url, old_pagepath, new_pagepath): + """Rewrite a Markdown URL pointing to a renamed page or one of its + attachments. + + Returns the rewritten URL, or the original URL if no rewrite is needed. + """ + + from otterwiki.helper import get_filename + + retain_case = app.config.get("RETAIN_PAGE_NAME_CASE", False) + + # Split query string. + path, qsep, query = url.partition("?") + + # Split anchor. + path, hsep, anchor = path.partition("#") + + # + # First: attachment URLs + # + old_page_dir = quote(old_pagepath.rsplit(".", 1)[0].strip("/"), safe="/") + new_page_dir = quote(new_pagepath.rsplit(".", 1)[0].strip("/"), safe="/") + + old_prefix = "/" + old_page_dir + "/" + + if retain_case: + matches = path.startswith(old_prefix) + else: + matches = path.lower().startswith(old_prefix.lower()) + + if matches: + relative = path[len(old_prefix) - 1 :] + new_path = "/" + new_page_dir + relative + + else: + # + # Page URL + # + leading_slash = path.startswith("/") + + page = unquote(path.lstrip("/")) + + if not page: + return url + + old_key = get_filename(old_pagepath) + if get_filename(page) != old_key: + return url + + encoded = quote(new_pagepath, safe="/") + new_path = ("/" if leading_slash else "") + encoded + + # Restore anchor. + if hsep: + new_path += "#" + anchor + + # Restore query string. + if qsep: + new_path += "?" + query + + return new_path + + +def _rewrite_content( + content, old_key, new_pagepath, linktitle_style, old_pagepath +): + def repl(match): + new_inner = _rewrite_inner( + match.group(1), old_key, new_pagepath, linktitle_style + ) + if new_inner is None: + return match.group(0) + return "[[" + new_inner + "]]" + + def wrapper_repl(match): + image = match.group(1) + url = _rewrite_url(match.group(2), old_pagepath, new_pagepath) + return f"[{image}]({url})" + + def attachment_repl(match): + return _rewrite_attachment(match, old_pagepath, new_pagepath) + + def markdown_link_repl(match): + return _rewrite_markdown_link( + match, + new_pagepath, + old_pagepath, + ) + + # Rewrite WikiLinks + content = WIKILINK_RE.sub(repl, content) + + # Rewrite nested images inside links + content = IMAGE_LINK_WRAPPER_RE.sub(wrapper_repl, content) + + # Rewrite image links + content = IMAGE_LINK_RE.sub(attachment_repl, content) + + # Rewrite Markdown links + content = MARKDOWN_LINK_RE.sub(markdown_link_repl, content) + + return content + + +def rename_backlinks(old_pagepath, new_pagepath): + from otterwiki.helper import get_filename + + old_key = get_filename(old_pagepath) + linktitle_style = _is_linktitle_style() + + all_files, _ = storage.list() + md_files = [f for f in all_files if f.endswith(".md")] + + updated = {} + for md_file in md_files: + try: + content = storage.load(md_file, mode="r") + new_content = _rewrite_content( + content, old_key, new_pagepath, linktitle_style, old_pagepath + ) + if new_content == content: + continue + # storage.update() writes the updated content to file. + storage.update( + filename=md_file, + content=new_content, + ) + updated[md_file] = new_content + except Exception as e: + app.logger.warning( + "Skipping page '%s': %s", + md_file, + e, + ) + continue + + if updated: + app.logger.info( + "rename_backlinks: rewrote backlinks in %d page(s) " + "after renaming '%s' to '%s'", + len(updated), + old_pagepath, + new_pagepath, + ) + + return updated diff --git a/otterwiki/gitstorage.py b/otterwiki/gitstorage.py index fbc94ee1..bd44ec0b 100644 --- a/otterwiki/gitstorage.py +++ b/otterwiki/gitstorage.py @@ -322,16 +322,12 @@ def log_slow(self, filename=None): # build and return logfile return [self._get_metadata_of_commit(commit) for commit in commits] - def store( + def update( self, filename, content, - message: str | None = "", - author=("", ""), mode="w", ): - if message is None: - message = "" dirname = os.path.dirname(filename) if dirname != "": os.makedirs( @@ -344,6 +340,22 @@ def store( diff = self.repo.index.diff(None, paths=filename) if len(diff) == 0 and filename not in self.repo.untracked_files: return False + + return True + + def store( + self, + filename, + content, + message: str | None = "", + author=("", ""), + mode="w", + ): + if not self.update(filename, content, mode): + return False + + if message is None: + message = "" # add and commit to git index = self.repo.index index.add([filename]) @@ -450,6 +462,7 @@ def rename( message=None, author=None, no_commit=False, + files_updated=[], ): if self.exists(new_filename): raise StorageError( @@ -472,7 +485,8 @@ def rename( if message is None: message = "{} renamed to {}.".format(old_filename, new_filename) if not no_commit: - self.commit([new_filename], message, author, no_add=True) + files_updated.append(new_filename) + self.commit(files_updated, message, author, no_add=True) def list(self, p=None, depth=None, exclude=[]): excludes = [".git"] + exclude diff --git a/otterwiki/help_admin.md b/otterwiki/help_admin.md index 76929ab6..f4081391 100644 --- a/otterwiki/help_admin.md +++ b/otterwiki/help_admin.md @@ -95,7 +95,7 @@ setting. Setting this to `optional` will allow empty commit messages. #### Page case name An Otter Wiki stores pages in files with names of all lowercase names. To retain -the upper and lower case of the filenames, check Retain page name case. +the upper and lower case of the filenames, check Retain page name case. #### Git Web server With Enable Git Server allow users with the permission to READ to clone and pull the wiki content via git and users with UPLOAD/Attachment management permissions to push content. HTTP Basic authentication is used for non anonymous access. There is no option for using git via ssh. When enabled, users find the URL to clone the repositroy in their settings. diff --git a/otterwiki/static/js/otterwiki.js b/otterwiki/static/js/otterwiki.js index 40cda1c6..2a23cad2 100644 --- a/otterwiki/static/js/otterwiki.js +++ b/otterwiki/static/js/otterwiki.js @@ -86,6 +86,21 @@ var otterwiki = { for (let i = 0; i < checkboxes.length; i++) { checkboxes[i].checked = !checkboxes[i].checked; } + }, + retain_update_backlinks_checkbox: function() { + const checkbox = document.getElementById("update_backlinks"); + if (!checkbox) return; + + const key = "otterwiki/rename/update-backlinks"; + + const saved = sessionStorage.getItem(key); + if (saved !== null) { + checkbox.checked = (saved === "true"); + } + + checkbox.addEventListener("change", () => { + sessionStorage.setItem(key, checkbox.checked); + }); } } @@ -269,6 +284,8 @@ window.addEventListener("keydown", function(event) { document.addEventListener("DOMContentLoaded", () => { + otterwiki.retain_update_backlinks_checkbox(); + if (document.body.dataset.pageIndexRetainUserExpandedNodes === 'False') return; const detailsList = document.querySelectorAll("details"); diff --git a/otterwiki/templates/rename.html b/otterwiki/templates/rename.html index 50d1a006..1c541870 100644 --- a/otterwiki/templates/rename.html +++ b/otterwiki/templates/rename.html @@ -65,6 +65,20 @@ +
+ +
+ + +
+
diff --git a/otterwiki/views.py b/otterwiki/views.py index e1551aac..30b5653c 100644 --- a/otterwiki/views.py +++ b/otterwiki/views.py @@ -534,6 +534,7 @@ def rename(path): new_pagename=request.form.get("new_pagename"), message=request.form.get("message"), author=otterwiki.auth.get_author(), + update_backlinks=request.form.get("update_backlinks"), ) return p.rename_form() diff --git a/otterwiki/wiki.py b/otterwiki/wiki.py index 9a219972..bbb0e0ce 100644 --- a/otterwiki/wiki.py +++ b/otterwiki/wiki.py @@ -73,6 +73,8 @@ int_or_None, ) +from .backlinks import rename_backlinks + # global timeout used in regexps _REGEX_TIMEOUT = 5 @@ -984,7 +986,7 @@ def history(self, rev_a: str | None = None, rev_b: str | None = None): breadcrumbs=self.breadcrumbs(), ) - def rename(self, new_pagename, message, author): + def rename(self, new_pagename, message, author, update_backlinks): if not has_permission("WRITE"): abort(403) # filename @@ -996,6 +998,14 @@ def rename(self, new_pagename, message, author): if not self.exists and (len(files) + len(directories)) == 0: self.exists_or_404() + pages_updated = ( + rename_backlinks(self.filename, new_pagename) + if update_backlinks + else {} + ) + assert pages_updated is not None + files_updated = list(pages_updated.keys()) + if (len(files) + len(directories)) > 0: # rename attachment directory new_attachment_directoryname = get_attachment_directoryname( @@ -1010,14 +1020,28 @@ def rename(self, new_pagename, message, author): # if self.exists, do not commit yet, commit will follow below, when the md file is renamed # if not self.exists, do commit, since no md file will be renamed no_commit=self.exists, + files_updated=files_updated, ) # rename page if self.exists: storage.rename( - self.filename, new_filename, message=message, author=author + self.filename, + new_filename, + message=message, + author=author, + files_updated=files_updated, + ) + + # notify plugins of backlink pages updated + for pagepath, content in pages_updated.items(): + plugin_manager.hook.page_saved( + pagepath=pagepath, + content=content, + author=author, + message=message, ) - def handle_rename(self, new_pagename, message, author): + def handle_rename(self, new_pagename, message, author, update_backlinks): if not has_permission("WRITE"): abort(403) if empty(new_pagename): @@ -1039,7 +1063,7 @@ def handle_rename(self, new_pagename, message, author): ) try: old_pagepath = self.pagepath - self.rename(new_pagename, message, author) + self.rename(new_pagename, message, author, update_backlinks) except Exception as e: # I tried to plumb an error message in here, but it did not show up in the UI - turns out these messages # get stored in the cookie, and some browsers don't like big cookies: diff --git a/tests/test_backlinks.py b/tests/test_backlinks.py new file mode 100644 index 00000000..4b1fbad8 --- /dev/null +++ b/tests/test_backlinks.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python +# vim: set et ts=8 sts=4 sw=4 ai: + +import pytest + + +def save_shortcut(test_client, pagename, content): + rv = test_client.post( + "/{}/save".format(pagename), + data={ + "content": content, + "commit": "test", + }, + follow_redirects=True, + ) + assert rv.status_code == 200 + + +@pytest.fixture +def app_with_default_backlink_settings(create_app): + create_app.config["WIKILINK_STYLE"] = "" + create_app.config["RETAIN_PAGE_NAME_CASE"] = False + return create_app + + +@pytest.fixture +def test_client(app_with_default_backlink_settings): + return app_with_default_backlink_settings.test_client() + + +@pytest.mark.parametrize( + "wikilink_style,content,expected", + [ + ( + "", + "[[Target Title|Path/OldPage]]", + "[[Target Title|Path/NewPage]]", + ), + ( + "LINKTITLE", + "[[Path/OldPage|Target Title]]", + "[[Path/NewPage|Target Title]]", + ), + ], +) +def test_rename_backlinks_wikilink_style( + create_app, + wikilink_style, + content, + expected, +): + app = create_app + saved = app.config["WIKILINK_STYLE"] + app.config["WIKILINK_STYLE"] = wikilink_style + + content += "\n" + expected += "\n" + + with app.test_client() as client: + save_shortcut(client, "example", content) + + from otterwiki.backlinks import rename_backlinks + + result = rename_backlinks( + "Path/OldPage", + "Path/NewPage", + ) + + assert result == { + "example.md": expected, + } + + updated = app.storage.load("example.md") + assert updated == expected + + app.config["WIKILINK_STYLE"] = saved + + +@pytest.mark.parametrize( + "retain_case,content,expected", + [ + ( + False, + "![](/path/oldpage/image.png)", + "![](/Path/NewPage/image.png)", + ), + ( + True, + "![](/path/oldpage/image.png)", + "![](/path/oldpage/image.png)", + ), + ], +) +def test_rename_backlinks_retain_page_name_case( + create_app, + retain_case, + content, + expected, +): + app = create_app + saved = app.config["RETAIN_PAGE_NAME_CASE"] + app.config["RETAIN_PAGE_NAME_CASE"] = retain_case + + content += "\n" + expected += "\n" + + with app.test_client() as client: + save_shortcut(client, "example", content) + + from otterwiki.backlinks import rename_backlinks + + result = rename_backlinks( + "Path/OldPage", + "Path/NewPage", + ) + + if expected != content: + assert result == { + "example.md": expected, + } + else: + assert result == {} + + assert app.storage.load("example.md") == expected + + app.config["RETAIN_PAGE_NAME_CASE"] = saved + + +@pytest.mark.parametrize( + "old_page,new_page,content,expected", + [ + ( + "Pathto/Targetpage", + "Pathto/Renamedpage", + "[[Pathto/Targetpage]]", + "[[Pathto/Renamedpage]]", + ), + ( + "Pathto/Targetpage", + "Pathto/Renamedpage", + "[[Target Page|Pathto/Targetpage]]", + "[[Target Page|Pathto/Renamedpage]]", + ), + ( + "Pathto/Targetpage", + "Pathto/Renamedpage", + "[Description](/Pathto/Targetpage)", + "[Description](/Pathto/Renamedpage)", + ), + ( + "Pathto/Targetpage", + "Pathto/Renamedpage", + "[[Pathto/Targetpage#sub-heading]]", + "[[Pathto/Renamedpage#sub-heading]]", + ), + ( + "Pathto/Targetpage", + "Pathto/Renamedpage", + "[[Target Page With Anchor|Pathto/Targetpage#sub-heading]]", + "[[Target Page With Anchor|Pathto/Renamedpage#sub-heading]]", + ), + ( + "Pathto/Targetpage", + "Pathto/Renamedpage", + "[Anchor](/Pathto/Targetpage#sub-heading)", + "[Anchor](/Pathto/Renamedpage#sub-heading)", + ), + ( + "Pathto/Targetpage", + "Pathto/Renamedpage", + "[Otter](/Pathto/Targetpage/otter.png)", + "[Otter](/Pathto/Renamedpage/otter.png)", + ), + ( + "Pathto/Targetpage", + "Pathto/Renamedpage", + "![](/Pathto/Targetpage/otter.png)", + "![](/Pathto/Renamedpage/otter.png)", + ), + ( + "Path With Spaces/Target Page", + "Path With Spaces/Renamed Page", + "[Description](/Path%20With%20Spaces/Target%20Page)", + "[Description](/Path%20With%20Spaces/Renamed%20Page)", + ), + ], +) +def test_rename_backlinks_supported_links( + create_app, + old_page, + new_page, + content, + expected, +): + app = create_app + + content += "\n" + expected += "\n" + + with app.test_client() as client: + save_shortcut(client, "example", content) + + from otterwiki.backlinks import rename_backlinks + + result = rename_backlinks( + old_page, + new_page, + ) + + assert result == { + "example.md": expected, + } + + assert app.storage.load("example.md") == expected diff --git a/tests/test_rename_with_updated_links.py b/tests/test_rename_with_updated_links.py new file mode 100644 index 00000000..ad9c5ab1 --- /dev/null +++ b/tests/test_rename_with_updated_links.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# vim: set et ts=8 sts=4 sw=4 ai: + +import re + +import pytest + + +@pytest.fixture +def test_client(create_app): + return create_app.test_client() + + +def save_shortcut(test_client, pagename, content, commit_message): + rv = test_client.post( + "/{}/save".format(pagename), + data={ + "content": content, + "commit": commit_message, + }, + follow_redirects=True, + ) + assert rv.status_code == 200 + + +def rename_shortcut(test_client, pagename, new_pagename): + rv = test_client.post( + "/{}/rename".format(pagename), + data={ + "new_pagename": new_pagename, + "message": "", + "update_backlinks": "1", + }, + follow_redirects=True, + ) + assert rv.status_code == 200 + + +def find_link_href(html, text): + """Return the href of the rendered anchor with the given link text, + or None if the link was not rendered as an anchor at all.""" + m = re.search( + r']*href="([^"]*)"[^>]*>{}'.format(re.escape(text)), + html, + ) + return m.group(1) if m else None + + +def test_rename_updates_percent_encoded_markdown_link(test_client): + """Markdown links to a page with a space in its name are commonly + written percent-encoded, e.g. [a link](/Target%20Page). Renaming + 'Target Page' must update these links, too.""" + save_shortcut( + test_client, + "Target Page", + "# Target Page\n", + "created target", + ) + save_shortcut( + test_client, + "Linker", + "# Linker\n\n[a link](/Target%20Page)\n", + "created linker", + ) + rename_shortcut(test_client, "Target Page", "RenamedTarget") + # the encoded link to the old name must be gone from the linking page + content = test_client.application.storage.load("linker.md") + assert "target%20page" not in content.lower() + # and the rendered link must resolve to the renamed page + html = test_client.get("/Linker/view").data.decode() + href = find_link_href(html, "a link") + assert href is not None, "link is no longer rendered as an anchor" + rv = test_client.get(href) + assert rv.status_code == 200 + assert "Target Page" in rv.data.decode() + + +def test_rename_to_name_with_space_keeps_link_parseable(test_client): + """When a page is renamed to a name containing a space, the + rewritten link destination must not contain a raw space: + [a link](/New Name) is not parsed as a markdown link.""" + save_shortcut( + test_client, + "Source", + "# Source\n", + "created source", + ) + save_shortcut( + test_client, + "Linker", + "# Linker\n\n[a link](/Source)\n", + "created linker", + ) + rename_shortcut(test_client, "Source", "New Name") + # a raw space in the link destination breaks the markdown link + content = test_client.application.storage.load("linker.md") + assert "](/New Name)" not in content + # the rendered link must still be an anchor and resolve to the + # renamed page + html = test_client.get("/Linker/view").data.decode() + href = find_link_href(html, "a link") + assert href is not None, "link is no longer rendered as an anchor" + rv = test_client.get(href) + assert rv.status_code == 200 + assert "Source" in rv.data.decode()