python-HyperKitty/mistune3.patch

From 2d123efddf474b6cc367b16e59ba9b99e95164e1 Mon Sep 17 00:00:00 2001
From: Abhilash Raj <raj.abhilash1@gmail.com>
Date: Wed, 5 Jul 2023 10:15:04 +0530
Subject: [PATCH 1/6] fix: Update mistune plugin to be compat with 3.0.x

Mistune bump to 3.x updated the plugin API making our current one
incompatible and requiring changes. This commit makes the required
changes so we can work with newer version.

This also strictly works on mistune>=3.0 since the older API won't
work for us anymore.
---
 hyperkitty/lib/haystack.py | 57 ++++++++++++++++++++++++++++++++++++++
 hyperkitty/lib/renderer.py | 44 +++++++++++++++--------------
 setup.py                   |  2 +-
 4 files changed, 82 insertions(+), 22 deletions(-)
 create mode 100644 hyperkitty/lib/haystack.py

diff --git a/hyperkitty/lib/haystack.py b/hyperkitty/lib/haystack.py
new file mode 100644
index 00000000..8dbe64a7
--- /dev/null
+++ b/hyperkitty/lib/haystack.py
@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2023 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+"""This module contains Django-haystack backend for SQLlite3."""
+
+from haystack.backends import (
+    BaseEngine, BaseSearchBackend, BaseSearchQuery, SearchNode, log_query)
+
+
+class SqliteSearchBackend(BaseSearchBackend):
+
+    def update(self, index, iterable, commit=True):
+        return super().update(index, iterable, commit)
+
+    def remove(self, obj_or_string):
+        return super().remove(obj_or_string)
+
+    def clear(self, models=None, commit=True):
+        return super().clear(models, commit)
+
+    @log_query
+    def search(self, query_string, **kwargs):
+        return super().search(query_string, **kwargs)
+
+    def prep_value(self, value):
+        return super().prep_value(value)
+
+    def more_like_this(self, model_instance, additional_query_string=None, result_class=None):
+        return super().more_like_this(model_instance, additional_query_string, result_class)
+
+
+class SqliteSearchQuery(BaseSearchQuery):
+
+    def build_query(self):
+        return super().build_query()
+
+
+class SqliteEngine(BaseEngine):
+
+    backend = SqliteSearchBackend
+    query = SqliteSearchQuery
\ No newline at end of file
diff --git a/hyperkitty/lib/renderer.py b/hyperkitty/lib/renderer.py
index cbb83b9a..2db2aee6 100644
--- a/hyperkitty/lib/renderer.py
+++ b/hyperkitty/lib/renderer.py
@@ -3,8 +3,9 @@ import re
 from django.conf import settings
 
 import mistune
-from mistune.plugins.extra import plugin_url
-from mistune.util import escape_html, escape_url
+from mistune.plugins.url import url
+from mistune.util import escape as escape_html
+from mistune.util import escape_url
 
 
 class MyRenderer(mistune.HTMLRenderer):
@@ -42,7 +43,7 @@ class MyRenderer(mistune.HTMLRenderer):
         return '![{alt}]({src} {title})'.format(
             src=src, title=title, alt=alt)
 
-    def image(self, src, alt_text, title):
+    def image(self, alt, src, title):
         """Render image if configured to do so.
 
         HYPERKITTY_RENDER_INLINE_IMAGE configuration allows for
@@ -50,10 +51,10 @@ class MyRenderer(mistune.HTMLRenderer):
         default since embeded images can cause problems.
         """
         if getattr(settings, 'HYPERKITTY_RENDER_INLINE_IMAGE', False):
-            return super().image(src, alt_text, title, )
-        return self._md_style_img(src, title, alt_text)
+            return super().image(src, alt, title, )
+        return self._md_style_img(src, title, alt)
 
-    def link(self, link, text=None, title=None):
+    def link(self, text=None, url=None, title=None):
         """URL link renderer that truncates the length of the URL.
 
         This only does it for the URLs that are not hyperlinks by just literal
@@ -61,14 +62,14 @@ class MyRenderer(mistune.HTMLRenderer):
         It also adds target=“_blank” so that the URLs open in a new tab.
         """
         if text is None:
-            text = link
+            text = url
             if len(text) > 76:
-                text = link[:76] + '...'
+                text = url[:76] + '...'
 
-        s = '<a target="_blank" href="' + self._safe_url(link) + '"'
+        s = '<a target="_blank" href="' + self.safe_url(url) + '"'
         if title:
             s += ' title="' + escape_html(title) + '"'
-        return s + '>' + (text or link) + '</a>'
+        return s + '>' + (text or url) + '</a>'
 
 
 class InlineParser(mistune.inline_parser.InlineParser):
@@ -113,7 +114,7 @@ def remove_header_rules(rules):
 class BlockParser(mistune.block_parser.BlockParser):
     """A copy of Mistune's block parser with header parsing rules removed."""
     RULE_NAMES = remove_header_rules(
-        mistune.block_parser.BlockParser.RULE_NAMES)
+        mistune.block_parser.BlockParser.DEFAULT_RULES)
 
 
 OUTLOOK_REPLY_PATTERN = re.compile(
@@ -163,10 +164,10 @@ def plugin_signature(md):
 
     It only provides an HTML renderer because that is the only one needed.
     """
-    md.block.register_rule('signature', SIGNATURE_PATTERN, parse_signature)
+    md.block.register('signature', SIGNATURE_PATTERN, parse_signature)
 
-    md.block.rules.insert(0,  'signature')
-    if md.renderer.NAME == 'html':
+    # md.block.rules.insert(0,  'signature')
+    if md.renderer and md.renderer.NAME == 'html':
         md.renderer.register('signature', render_html_signature)
 
 
@@ -211,22 +212,23 @@ def plugin_pgp_signature(md):
     It parses BEGIN PGP SIGNATURE and END PGP SIGNATURE and collapses content
     in between them.
     """
-    md.block.register_rule('pgp', PGP_SIGNATURE_MATCH, parse_pgp_signature)
-    md.block.rules.append('pgp')
-    if md.renderer.NAME == 'html':
+    md.block.register('pgp', PGP_SIGNATURE_MATCH, parse_pgp_signature)
+    # md.block.rules.append('pgp')
+    if md.renderer and md.renderer.NAME == 'html':
         md.renderer.register('pgp', render_pgp_signature)
 
 
 renderer = MyRenderer(escape=True)
 markdown_renderer = mistune.Markdown(
     renderer=renderer,
-    inline=InlineParser(renderer, hard_wrap=False),
+    inline=InlineParser(hard_wrap=False),
     block=BlockParser(),
+
     plugins=[
         plugin_pgp_signature,
         plugin_signature,
         plugin_outlook_reply,
-        plugin_url
+        url
         ])
 
 
@@ -235,10 +237,10 @@ markdown_renderer = mistune.Markdown(
 # rules that results in a regularly formatted email.
 text_renderer = mistune.Markdown(
     renderer=renderer,
-    inline=InlineParser(renderer, hard_wrap=False),
+    inline=InlineParser(hard_wrap=False),
     block=BlockParser(),
     plugins=[plugin_disable_markdown,
              plugin_pgp_signature,
              plugin_signature,
-             plugin_url,
+             url,
              ])
diff --git a/setup.py b/setup.py
index 2d8da168..d8c2d2fb 100755
--- a/setup.py
+++ b/setup.py
@@ -45,7 +45,7 @@ REQUIRES = [
     "pytz>=2012",
     "django-compressor>=1.3",
     "mailmanclient>=3.3.3",
-    "mistune>=2.0.0,<3.0",
+    "mistune>=3.0",
     "python-dateutil >= 2.0",
     "networkx>=2.0",
     "django-haystack>=2.8.0",
-- 
GitLab


From 2312689ea67f631c2aae54013a3e6a57be7ef36d Mon Sep 17 00:00:00 2001
From: Abhilash Raj <raj.abhilash1@gmail.com>
Date: Wed, 5 Jul 2023 10:20:04 +0530
Subject: [PATCH 2/6] Remove the hack for urls with @ since it is fixed
 upstream

---
 hyperkitty/lib/renderer.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/hyperkitty/lib/renderer.py b/hyperkitty/lib/renderer.py
index 2db2aee6..6a86cc4c 100644
--- a/hyperkitty/lib/renderer.py
+++ b/hyperkitty/lib/renderer.py
@@ -87,21 +87,6 @@ class InlineParser(mistune.inline_parser.InlineParser):
             return 'emphasis', marker, self.render(text, state)
         return 'strong', marker, self.render(text, state)
 
-    # This is an override for a fix that should be in mistune.
-    # https://github.com/lepture/mistune/pull/276
-    def parse_auto_link(self, m, state):
-        if state.get('_in_link'):
-            return 'text', m.group(0)
-
-        text = m.group(1)
-        if ('@' in text and
-            not text.lower().startswith('mailto:') and
-                not text.lower().startswith('http')):
-            link = 'mailto:' + text
-        else:
-            link = text
-        return 'link', escape_url(link), text
-
 
 def remove_header_rules(rules):
     rules = list(rules)
-- 
GitLab


From 373dadee496c70ee0c6384ef2ad467919e5315cb Mon Sep 17 00:00:00 2001
From: Abhilash Raj <raj.abhilash1@gmail.com>
Date: Wed, 5 Jul 2023 10:39:31 +0530
Subject: [PATCH 3/6] QA checks

---
 hyperkitty/lib/renderer.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hyperkitty/lib/renderer.py b/hyperkitty/lib/renderer.py
index 6a86cc4c..b233dcad 100644
--- a/hyperkitty/lib/renderer.py
+++ b/hyperkitty/lib/renderer.py
@@ -5,7 +5,6 @@ from django.conf import settings
 import mistune
 from mistune.plugins.url import url
 from mistune.util import escape as escape_html
-from mistune.util import escape_url
 
 
 class MyRenderer(mistune.HTMLRenderer):
-- 
GitLab


From 481e81d32c2e5ff1c9516fe41512bcfd3cf2c208 Mon Sep 17 00:00:00 2001
From: Abhilash Raj <raj.abhilash1@gmail.com>
Date: Wed, 5 Jul 2023 13:10:49 +0530
Subject: [PATCH 4/6] Fixes for more mistune 3.x compat

---
 hyperkitty/lib/renderer.py            | 32 ++++++++++++++-------------
 hyperkitty/tests/test_templatetags.py |  3 +++
 2 files changed, 20 insertions(+), 15 deletions(-)

diff --git a/hyperkitty/lib/renderer.py b/hyperkitty/lib/renderer.py
index b233dcad..346472d2 100644
--- a/hyperkitty/lib/renderer.py
+++ b/hyperkitty/lib/renderer.py
@@ -5,6 +5,7 @@ from django.conf import settings
 import mistune
 from mistune.plugins.url import url
 from mistune.util import escape as escape_html
+from mistune.util import safe_entity
 
 
 class MyRenderer(mistune.HTMLRenderer):
@@ -24,15 +25,16 @@ class MyRenderer(mistune.HTMLRenderer):
         quote-switched classed hyperlink that can collapse the next quote-text
         using JS.
         """
+        print(text)
         return (
             f'<div class="quoted-switch"><a href="#">...</a></div>'
             f'<blockquote class="blockquote quoted-text">{text}</blockquote>')
 
-    def emphasis(self, marker, text):
+    def emphasis(self, text, marker):
         """Emphasis with marker included."""
         return super().emphasis(marker + text + marker)
 
-    def strong(self, marker, text):
+    def strong(self, text, marker):
         """Strong with marker included."""
         return super().strong(marker + text + marker)
 
@@ -42,7 +44,7 @@ class MyRenderer(mistune.HTMLRenderer):
         return '![{alt}]({src} {title})'.format(
             src=src, title=title, alt=alt)
 
-    def image(self, alt, src, title):
+    def image(self, alt, url, title=None):
         """Render image if configured to do so.
 
         HYPERKITTY_RENDER_INLINE_IMAGE configuration allows for
@@ -50,10 +52,10 @@ class MyRenderer(mistune.HTMLRenderer):
         default since embeded images can cause problems.
         """
         if getattr(settings, 'HYPERKITTY_RENDER_INLINE_IMAGE', False):
-            return super().image(src, alt, title, )
-        return self._md_style_img(src, title, alt)
+            return super().image(alt, url, title)
+        return self._md_style_img(url, title, alt)
 
-    def link(self, text=None, url=None, title=None):
+    def link(self, text, url, title=None):
         """URL link renderer that truncates the length of the URL.
 
         This only does it for the URLs that are not hyperlinks by just literal
@@ -67,7 +69,7 @@ class MyRenderer(mistune.HTMLRenderer):
 
         s = '<a target="_blank" href="' + self.safe_url(url) + '"'
         if title:
-            s += ' title="' + escape_html(title) + '"'
+            s += ' title="' + safe_entity(title) + '"'
         return s + '>' + (text or url) + '</a>'
 
 
@@ -79,13 +81,13 @@ class InlineParser(mistune.inline_parser.InlineParser):
     ‘emphasis’ or ‘strong’ node.
     """
 
-    def tokenize_emphasis(self, m, state):
-        marker = m.group(1)
-        text = m.group(2)
-        if len(marker) == 1:
-            return 'emphasis', marker, self.render(text, state)
-        return 'strong', marker, self.render(text, state)
-
+    def parse_emphasis(self, m, state):
+        end_pos = super().parse_emphasis(m, state)
+        last_token = state.tokens[-1].copy()
+        marker = m.group(0)
+        last_token['attrs'] = {'marker': marker}
+        state.tokens[-1] = last_token
+        return end_pos
 
 def remove_header_rules(rules):
     rules = list(rules)
@@ -97,7 +99,7 @@ def remove_header_rules(rules):
 
 class BlockParser(mistune.block_parser.BlockParser):
     """A copy of Mistune's block parser with header parsing rules removed."""
-    RULE_NAMES = remove_header_rules(
+    DEFAULT_RULES = remove_header_rules(
         mistune.block_parser.BlockParser.DEFAULT_RULES)
 
 
diff --git a/hyperkitty/tests/test_templatetags.py b/hyperkitty/tests/test_templatetags.py
index 79372d84..b1d83177 100644
--- a/hyperkitty/tests/test_templatetags.py
+++ b/hyperkitty/tests/test_templatetags.py
@@ -156,6 +156,7 @@ On Fri, 09.11.12 11:27, Someone wrote:
 > This is the first quoted line
 > On Fri 07.25.12, Aperson wrote:
 >> This is the second quoted line.
+
 This is the response.
 """
         expected = (
@@ -167,6 +168,8 @@ This is the response.
             '</blockquote></blockquote>'
             '<p>This is the response.</p>\n')
         result = markdown_renderer(contents)
+        print('----------'*10)
+        print(result)
         self.assertEqual(result.strip(), expected.strip())
 
     def test_parse_heading_normal(self):
-- 
GitLab


From b2d809b446402296b5e1b830a7f408a6e464210d Mon Sep 17 00:00:00 2001
From: Abhilash Raj <raj.abhilash1@gmail.com>
Date: Wed, 12 Jul 2023 12:23:23 +0530
Subject: [PATCH 5/6] Fix the outlook_reply_plugin for mixtune 3.x

---
 hyperkitty/lib/renderer.py            | 29 ++++++++++++++-------------
 hyperkitty/tests/test_templatetags.py | 12 +++--------
 2 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/hyperkitty/lib/renderer.py b/hyperkitty/lib/renderer.py
index 346472d2..ddb2a77d 100644
--- a/hyperkitty/lib/renderer.py
+++ b/hyperkitty/lib/renderer.py
@@ -25,7 +25,6 @@ class MyRenderer(mistune.HTMLRenderer):
         quote-switched classed hyperlink that can collapse the next quote-text
         using JS.
         """
-        print(text)
         return (
             f'<div class="quoted-switch"><a href="#">...</a></div>'
             f'<blockquote class="blockquote quoted-text">{text}</blockquote>')
@@ -62,7 +61,10 @@ class MyRenderer(mistune.HTMLRenderer):
         URLs (text=None) so text is same as URL.
         It also adds target=“_blank” so that the URLs open in a new tab.
         """
-        if text is None:
+        # text can be none of same as url in case of autolink parsing. This
+        # will truncate the length of the URL in both cases but preserve
+        # the actual URL destination in the hyperlink.
+        if text is None or text == url:
             text = url
             if len(text) > 76:
                 text = url[:76] + '...'
@@ -103,26 +105,26 @@ class BlockParser(mistune.block_parser.BlockParser):
         mistune.block_parser.BlockParser.DEFAULT_RULES)
 
 
-OUTLOOK_REPLY_PATTERN = re.compile(
+OUTLOOK_REPLY_PATTERN = (
     r'^-------- Original message --------\n'
-    r'([\s\S]+)',                             # everything after newline.
-    re.M
+    r'(?P<reply_text>[\s\S]+)'                             # everything after newline
 )
 
 
 def parse_outlook_reply(block, m, state):
     """Parser for outlook style replies."""
-    text = m.group(0)
-    return {
+    text = m.group('reply_text')
+    reply_token = '-------- Original message --------\n'
+    state.append_token({
         'type': 'block_quote',
-        'children': [{'type': 'paragraph', 'text': text}]
-        }
+        'children': [{'type': 'paragraph', 'text': reply_token + text}],
+        })
+    return m.end() + 1
 
 
 def plugin_outlook_reply(md):
-    md.block.register_rule(
+    md.block.register(
         'outlook_reply', OUTLOOK_REPLY_PATTERN, parse_outlook_reply)
-    md.block.rules.insert(-1,  'outlook_reply')
 
 
 # Signature Plugin looks for signature pattern in email content and converts it
@@ -209,12 +211,11 @@ markdown_renderer = mistune.Markdown(
     renderer=renderer,
     inline=InlineParser(hard_wrap=False),
     block=BlockParser(),
-
     plugins=[
+        plugin_outlook_reply,
         plugin_pgp_signature,
         plugin_signature,
-        plugin_outlook_reply,
-        url
+        url,
         ])
 
 
diff --git a/hyperkitty/tests/test_templatetags.py b/hyperkitty/tests/test_templatetags.py
index b1d83177..f567388d 100644
--- a/hyperkitty/tests/test_templatetags.py
+++ b/hyperkitty/tests/test_templatetags.py
@@ -147,9 +147,6 @@ class TestGravatar(TestCase):
 
 class TestDecorate(TestCase):
 
-    def setUp(self):
-        pass
-
     def test_parse_quote(self):
         contents = """
 On Fri, 09.11.12 11:27, Someone wrote:
@@ -168,8 +165,6 @@ This is the response.
             '</blockquote></blockquote>'
             '<p>This is the response.</p>\n')
         result = markdown_renderer(contents)
-        print('----------'*10)
-        print(result)
         self.assertEqual(result.strip(), expected.strip())
 
     def test_parse_heading_normal(self):
@@ -187,7 +182,7 @@ https://some.url/llasdfjaksdgfjsdfgkjasdfbgksdfjgbsdfkgjbsdflkgjbsdflgksjdhfbgks
         result = markdown_renderer(contents)
         self.assertEqual(
             result.strip(),
-            '<p><a target="_blank" href="https://some.url/llasdfjaksdgfjsdfgkjasdfbgksdfjgbsdfkgjbsdflkgjbsdflgksjdhfbgksdfgb">https://some.url/llasdfjaksdgfjsdfgkjasdfbgksdfjgbsdfkgjbsdflkgjbsdflgksjdhf...</a></p>')   # noqa: E501
+            ('<p><a target="_blank" href="https://some.url/llasdfjaksdgfjsdfgkjasdfbgksdfjgbsdfkgjbsdflkgjbsdflgksjdhfbgksdfgb">https://some.url/llasdfjaksdgfjsdfgkjasdfbgksdfjgbsdfkgjbsdflkgjbsdflgksjdhf...</a></p>'))   # noqa: E501
 
     def test_autolink_small_url(self):
         # Test that autolink doesn't add ... to URLs that aren't truncated.
@@ -241,6 +236,7 @@ Subject: Testing if the quoted reply works with Outlook style.
 This is the original text *with* some __markup__.
 """
         result = markdown_renderer(contents)
+        print(result)
         self.assertEqual(
             result.strip(),
             """<p>This is the replied text.</p>
@@ -250,9 +246,7 @@ From: A person &lt;person(a)example.com&gt;
 Date: 6/26/23 16:23 (GMT-05:00)
 To: mytestlist@example.com
 Subject: Testing if the quoted reply works with Outlook style.
-
-This is the original text <em>*with*</em> some <strong>__markup__</strong>.
-</p>
+This is the original text <em>*with*</em> some <strong>__markup__</strong>.</p>
 </blockquote>""")  # noqa: E501
 
 
-- 
GitLab


From af17c334a2a35d672c9c732ed9a37bc780fdb64d Mon Sep 17 00:00:00 2001
From: Abhilash Raj <raj.abhilash1@gmail.com>
Date: Wed, 12 Jul 2023 14:14:00 +0530
Subject: [PATCH 6/6] qa fixes

---
 hyperkitty/lib/renderer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/hyperkitty/lib/renderer.py b/hyperkitty/lib/renderer.py
index ddb2a77d..7f1a3230 100644
--- a/hyperkitty/lib/renderer.py
+++ b/hyperkitty/lib/renderer.py
@@ -4,7 +4,6 @@ from django.conf import settings
 
 import mistune
 from mistune.plugins.url import url
-from mistune.util import escape as escape_html
 from mistune.util import safe_entity
 
 
@@ -91,6 +90,7 @@ class InlineParser(mistune.inline_parser.InlineParser):
         state.tokens[-1] = last_token
         return end_pos
 
+
 def remove_header_rules(rules):
     rules = list(rules)
     for rule in ('setex_header', 'axt_heading'):
@@ -107,7 +107,7 @@ class BlockParser(mistune.block_parser.BlockParser):
 
 OUTLOOK_REPLY_PATTERN = (
     r'^-------- Original message --------\n'
-    r'(?P<reply_text>[\s\S]+)'                             # everything after newline
+    r'(?P<reply_text>[\s\S]+)'                    # everything after newline
 )
 
 
-- 
GitLab