Heading markdown fix (#908)

* Copy heading parser and adjust it to match Reddit behavior

Unlike CommonMark, Reddit does not require space after #. This behavior is
coded in a private static function, so the only way to override it is to
copy everything and use the modified copy instead of the default parser.

* Use RedditHeadingPlugin instead of regexes

* Apply plugins to post body when writing a comment

This fixes display when writing comment to a post
that contains spoilers or headings without space

* Apply plugins to parent comment body when writing a comment

This fixes display when replying to a comment that contains strikethrough text

app/src/main/java/ml/docilealligator/infinityforreddit/activities/CommentActivity.java

@@ -82,6 +82,7 @@ import ml.docilealligator.infinityforreddit.comment.SendComment;
 import ml.docilealligator.infinityforreddit.customtheme.CustomThemeWrapper;
 import ml.docilealligator.infinityforreddit.customviews.LinearLayoutManagerBugFixed;
 import ml.docilealligator.infinityforreddit.events.SwitchAccountEvent;
+import ml.docilealligator.infinityforreddit.markdown.RedditHeadingPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SpoilerParserPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SuperscriptInlineProcessor;
 import ml.docilealligator.infinityforreddit.utils.SharedPreferencesUtils;
@@ -162,6 +163,7 @@ public class CommentActivity extends BaseActivity implements UploadImageEnabledA
     private boolean isSubmitting = false;
     private boolean isReplying;
     private int markdownColor;
+    private int spoilerBackgroundColor;
     private Uri capturedImageUri;
     private ArrayList<UploadedImage> uploadedImages = new ArrayList<>();
     private Menu mMenu;
@@ -233,6 +235,8 @@ public class CommentActivity extends BaseActivity implements UploadImageEnabledA
                     }
                 })
                 .usePlugin(SpoilerParserPlugin.create(commentColor, commentSpoilerBackgroundColor))
+                .usePlugin(RedditHeadingPlugin.create())
+                .usePlugin(StrikethroughPlugin.create())
                 .usePlugin(LinkifyPlugin.create(Linkify.WEB_URLS))
                 .build();
         if (parentTextMarkdown != null) {
@@ -305,6 +309,8 @@ public class CommentActivity extends BaseActivity implements UploadImageEnabledA
                             builder.linkColor(linkColor);
                         }
                     })
+                    .usePlugin(SpoilerParserPlugin.create(markdownColor, spoilerBackgroundColor))
+                    .usePlugin(RedditHeadingPlugin.create())
                     .usePlugin(StrikethroughPlugin.create())
                     .usePlugin(LinkifyPlugin.create(Linkify.WEB_URLS))
                     .usePlugin(TableEntryPlugin.create(this))
@@ -434,6 +440,7 @@ public class CommentActivity extends BaseActivity implements UploadImageEnabledA
         int secondaryTextColor = mCustomThemeWrapper.getSecondaryTextColor();
         commentEditText.setHintTextColor(secondaryTextColor);
         markdownColor = secondaryTextColor;
+        spoilerBackgroundColor = markdownColor | 0xFF000000;
         accountNameTextView.setTextColor(mCustomThemeWrapper.getPrimaryTextColor());
 
         if (typeface != null) {

app/src/main/java/ml/docilealligator/infinityforreddit/activities/FullMarkdownActivity.java

@@ -54,6 +54,7 @@ import ml.docilealligator.infinityforreddit.customtheme.CustomThemeWrapper;
 import ml.docilealligator.infinityforreddit.customviews.LinearLayoutManagerBugFixed;
 import ml.docilealligator.infinityforreddit.customviews.MarkwonLinearLayoutManager;
 import ml.docilealligator.infinityforreddit.events.SwitchAccountEvent;
+import ml.docilealligator.infinityforreddit.markdown.RedditHeadingPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SpoilerParserPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SuperscriptInlineProcessor;
 import ml.docilealligator.infinityforreddit.utils.SharedPreferencesUtils;
@@ -169,6 +170,7 @@ public class FullMarkdownActivity extends BaseActivity {
                     }
                 })
                 .usePlugin(SpoilerParserPlugin.create(markdownColor, spoilerBackgroundColor))
+                .usePlugin(RedditHeadingPlugin.create())
                 .usePlugin(StrikethroughPlugin.create())
                 .usePlugin(LinkifyPlugin.create(Linkify.WEB_URLS))
                 .usePlugin(TableEntryPlugin.create(this))

app/src/main/java/ml/docilealligator/infinityforreddit/activities/WikiActivity.java

@@ -64,6 +64,7 @@ import ml.docilealligator.infinityforreddit.customtheme.CustomThemeWrapper;
 import ml.docilealligator.infinityforreddit.customviews.LinearLayoutManagerBugFixed;
 import ml.docilealligator.infinityforreddit.customviews.MarkwonLinearLayoutManager;
 import ml.docilealligator.infinityforreddit.events.SwitchAccountEvent;
+import ml.docilealligator.infinityforreddit.markdown.RedditHeadingPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SpoilerParserPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SuperscriptInlineProcessor;
 import ml.docilealligator.infinityforreddit.utils.JSONUtils;
@@ -198,6 +199,7 @@ public class WikiActivity extends BaseActivity {
                     }
                 })
                 .usePlugin(SpoilerParserPlugin.create(markdownColor, spoilerBackgroundColor))
+                .usePlugin(RedditHeadingPlugin.create())
                 .usePlugin(StrikethroughPlugin.create())
                 .usePlugin(MovementMethodPlugin.create(BetterLinkMovementMethod.linkify(Linkify.WEB_URLS).setOnLinkLongClickListener((textView, url) -> {
                     UrlMenuBottomSheetFragment urlMenuBottomSheetFragment = new UrlMenuBottomSheetFragment();

app/src/main/java/ml/docilealligator/infinityforreddit/adapters/CommentsListingRecyclerViewAdapter.java

@@ -68,6 +68,7 @@ import ml.docilealligator.infinityforreddit.customviews.CustomMarkwonAdapter;
 import ml.docilealligator.infinityforreddit.customviews.LinearLayoutManagerBugFixed;
 import ml.docilealligator.infinityforreddit.customviews.MarkwonLinearLayoutManager;
 import ml.docilealligator.infinityforreddit.customviews.SpoilerOnClickTextView;
+import ml.docilealligator.infinityforreddit.markdown.RedditHeadingPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SpoilerParserPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SuperscriptInlineProcessor;
 import ml.docilealligator.infinityforreddit.utils.APIUtils;
@@ -192,6 +193,7 @@ public class CommentsListingRecyclerViewAdapter extends PagedListAdapter<Comment
                     }
                 })
                 .usePlugin(SpoilerParserPlugin.create(mCommentColor, commentSpoilerBackgroundColor))
+                .usePlugin(RedditHeadingPlugin.create())
                 .usePlugin(StrikethroughPlugin.create())
                 .usePlugin(MovementMethodPlugin.create(BetterLinkMovementMethod.linkify(Linkify.WEB_URLS).setOnLinkLongClickListener((textView, url) -> {
                     if (!activity.isDestroyed() && !activity.isFinishing()) {

app/src/main/java/ml/docilealligator/infinityforreddit/adapters/CommentsRecyclerViewAdapter.java

@@ -78,6 +78,7 @@ import ml.docilealligator.infinityforreddit.customviews.LinearLayoutManagerBugFi
 import ml.docilealligator.infinityforreddit.customviews.MarkwonLinearLayoutManager;
 import ml.docilealligator.infinityforreddit.customviews.SpoilerOnClickTextView;
 import ml.docilealligator.infinityforreddit.fragments.ViewPostDetailFragment;
+import ml.docilealligator.infinityforreddit.markdown.RedditHeadingPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SpoilerParserPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SuperscriptInlineProcessor;
 import ml.docilealligator.infinityforreddit.post.Post;
@@ -224,6 +225,7 @@ public class CommentsRecyclerViewAdapter extends RecyclerView.Adapter<RecyclerVi
                     }
                 })
                 .usePlugin(SpoilerParserPlugin.create(mCommentTextColor, commentSpoilerBackgroundColor))
+                .usePlugin(RedditHeadingPlugin.create())
                 .usePlugin(StrikethroughPlugin.create())
                 .usePlugin(MovementMethodPlugin.create(BetterLinkMovementMethod.linkify(Linkify.WEB_URLS).setOnLinkLongClickListener((textView, url) -> {
                     if (!activity.isDestroyed() && !activity.isFinishing()) {

app/src/main/java/ml/docilealligator/infinityforreddit/adapters/MessageRecyclerViewAdapter.java

@@ -43,6 +43,7 @@ import ml.docilealligator.infinityforreddit.activities.ViewPrivateMessagesActivi
 import ml.docilealligator.infinityforreddit.activities.ViewUserDetailActivity;
 import ml.docilealligator.infinityforreddit.customtheme.CustomThemeWrapper;
 import ml.docilealligator.infinityforreddit.events.ChangeInboxCountEvent;
+import ml.docilealligator.infinityforreddit.markdown.RedditHeadingPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SpoilerParserPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SuperscriptInlineProcessor;
 import ml.docilealligator.infinityforreddit.message.FetchMessage;
@@ -135,6 +136,7 @@ public class MessageRecyclerViewAdapter extends PagedListAdapter<Message, Recycl
                     }
                 })
                 .usePlugin(SpoilerParserPlugin.create(mSecondaryTextColor, spoilerBackgroundColor))
+                .usePlugin(RedditHeadingPlugin.create())
                 .usePlugin(StrikethroughPlugin.create())
                 .usePlugin(LinkifyPlugin.create(Linkify.WEB_URLS))
                 .build();

app/src/main/java/ml/docilealligator/infinityforreddit/adapters/PostDetailRecyclerViewAdapter.java

@@ -110,6 +110,7 @@ import ml.docilealligator.infinityforreddit.customtheme.CustomThemeWrapper;
 import ml.docilealligator.infinityforreddit.customviews.AspectRatioGifImageView;
 import ml.docilealligator.infinityforreddit.customviews.MarkwonLinearLayoutManager;
 import ml.docilealligator.infinityforreddit.fragments.ViewPostDetailFragment;
+import ml.docilealligator.infinityforreddit.markdown.RedditHeadingPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SpoilerParserPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SuperscriptInlineProcessor;
 import ml.docilealligator.infinityforreddit.post.Post;
@@ -292,6 +293,7 @@ public class PostDetailRecyclerViewAdapter extends RecyclerView.Adapter<Recycler
                     }
                 })
                 .usePlugin(SpoilerParserPlugin.create(markdownColor, postSpoilerBackgroundColor))
+                .usePlugin(RedditHeadingPlugin.create())
                 .usePlugin(StrikethroughPlugin.create())
                 .usePlugin(MovementMethodPlugin.create(BetterLinkMovementMethod.linkify(Linkify.WEB_URLS).setOnLinkLongClickListener((textView, url) -> {
                     if (activity != null && !activity.isDestroyed() && !activity.isFinishing()) {

app/src/main/java/ml/docilealligator/infinityforreddit/adapters/PrivateMessagesDetailRecyclerViewAdapter.java

@@ -45,6 +45,7 @@ import ml.docilealligator.infinityforreddit.activities.LinkResolverActivity;
 import ml.docilealligator.infinityforreddit.activities.ViewPrivateMessagesActivity;
 import ml.docilealligator.infinityforreddit.activities.ViewUserDetailActivity;
 import ml.docilealligator.infinityforreddit.customtheme.CustomThemeWrapper;
+import ml.docilealligator.infinityforreddit.markdown.RedditHeadingPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SpoilerParserPlugin;
 import ml.docilealligator.infinityforreddit.markdown.SuperscriptInlineProcessor;
 import ml.docilealligator.infinityforreddit.message.Message;
@@ -119,6 +120,7 @@ public class PrivateMessagesDetailRecyclerViewAdapter extends RecyclerView.Adapt
                 })
                 .usePlugin(StrikethroughPlugin.create())
                 .usePlugin(SpoilerParserPlugin.create(commentColor, commentColor | 0xFF000000))
+                .usePlugin(RedditHeadingPlugin.create())
                 .usePlugin(LinkifyPlugin.create(Linkify.WEB_URLS))
                 .build();
         mShowElapsedTime = sharedPreferences.getBoolean(SharedPreferencesUtils.SHOW_ELAPSED_TIME_KEY, false);

app/src/main/java/ml/docilealligator/infinityforreddit/markdown/RedditHeadingParser.java

@@ -0,0 +1,118 @@
+package ml.docilealligator.infinityforreddit.markdown;
+
+import org.commonmark.internal.util.Parsing;
+import org.commonmark.node.Block;
+import org.commonmark.node.Heading;
+import org.commonmark.parser.InlineParser;
+import org.commonmark.parser.block.*;
+
+/**
+ * This is a copy of {@link org.commonmark.internal.HeadingParser} with a parsing change
+ * in {@link #getAtxHeading} to account for differences between Reddit and CommonMark
+ */
+public class RedditHeadingParser extends AbstractBlockParser {
+
+    private final Heading block = new Heading();
+    private final String content;
+
+    public RedditHeadingParser(int level, String content) {
+        block.setLevel(level);
+        this.content = content;
+    }
+
+    @Override
+    public Block getBlock() {
+        return block;
+    }
+
+    @Override
+    public BlockContinue tryContinue(ParserState parserState) {
+        // In both ATX and Setext headings, once we have the heading markup, there's nothing more to parse.
+        return BlockContinue.none();
+    }
+
+    @Override
+    public void parseInlines(InlineParser inlineParser) {
+        inlineParser.parse(content, block);
+    }
+
+    public static class Factory extends AbstractBlockParserFactory {
+
+        @Override
+        public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) {
+            if (state.getIndent() >= Parsing.CODE_BLOCK_INDENT) {
+                return BlockStart.none();
+            }
+
+            CharSequence line = state.getLine();
+            int nextNonSpace = state.getNextNonSpaceIndex();
+            RedditHeadingParser atxHeading = getAtxHeading(line, nextNonSpace);
+            if (atxHeading != null) {
+                return BlockStart.of(atxHeading).atIndex(line.length());
+            }
+
+            int setextHeadingLevel = getSetextHeadingLevel(line, nextNonSpace);
+            if (setextHeadingLevel > 0) {
+                CharSequence paragraph = matchedBlockParser.getParagraphContent();
+                if (paragraph != null) {
+                    String content = paragraph.toString();
+                    return BlockStart.of(new RedditHeadingParser(setextHeadingLevel, content))
+                            .atIndex(line.length())
+                            .replaceActiveBlockParser();
+                }
+            }
+
+            return BlockStart.none();
+        }
+    }
+
+    // spec: An ATX heading consists of a string of characters, parsed as inline content, between an opening sequence of
+    // 1–6 unescaped # characters and an optional closing sequence of any number of unescaped # characters.
+    // The optional closing sequence of #s must be preceded by a space and may be followed by spaces only.
+    //
+    // Unlike CommonMark, the opening sequence of # characters does not have to be followed by a space or by the end of line.
+    private static RedditHeadingParser getAtxHeading(CharSequence line, int index) {
+        int level = Parsing.skip('#', line, index, line.length()) - index;
+
+        if (level == 0 || level > 6) {
+            return null;
+        }
+
+        int start = index + level;
+        if (start >= line.length()) {
+            // End of line after markers is an empty heading
+            return new RedditHeadingParser(level, "");
+        }
+
+        int beforeSpace = Parsing.skipSpaceTabBackwards(line, line.length() - 1, start);
+        int beforeHash = Parsing.skipBackwards('#', line, beforeSpace, start);
+        int beforeTrailer = Parsing.skipSpaceTabBackwards(line, beforeHash, start);
+        if (beforeTrailer != beforeHash) {
+            return new RedditHeadingParser(level, line.subSequence(start, beforeTrailer + 1).toString());
+        } else {
+            return new RedditHeadingParser(level, line.subSequence(start, beforeSpace + 1).toString());
+        }
+    }
+
+    // spec: A setext heading underline is a sequence of = characters or a sequence of - characters, with no more than
+    // 3 spaces indentation and any number of trailing spaces.
+    private static int getSetextHeadingLevel(CharSequence line, int index) {
+        switch (line.charAt(index)) {
+            case '=':
+                if (isSetextHeadingRest(line, index + 1, '=')) {
+                    return 1;
+                }
+            case '-':
+                if (isSetextHeadingRest(line, index + 1, '-')) {
+                    return 2;
+                }
+        }
+        return 0;
+    }
+
+    private static boolean isSetextHeadingRest(CharSequence line, int index, char marker) {
+        int afterMarker = Parsing.skip(marker, line, index, line.length());
+        int afterSpace = Parsing.skipSpaceTab(line, afterMarker, line.length());
+        return afterSpace >= line.length();
+    }
+}

app/src/main/java/ml/docilealligator/infinityforreddit/markdown/RedditHeadingPlugin.java

@@ -0,0 +1,23 @@
+package ml.docilealligator.infinityforreddit.markdown;
+
+import androidx.annotation.NonNull;
+
+import org.commonmark.parser.Parser;
+
+import io.noties.markwon.AbstractMarkwonPlugin;
+
+/**
+ * Replaces CommonMark heading parsing with Reddit-style parsing that does not require space after #
+ */
+public class RedditHeadingPlugin extends AbstractMarkwonPlugin {
+
+    @NonNull
+    public static RedditHeadingPlugin create() {
+        return new RedditHeadingPlugin();
+    }
+
+    @Override
+    public void configureParser(@NonNull Parser.Builder builder) {
+        builder.customBlockParserFactory(new RedditHeadingParser.Factory());
+    }
+}

app/src/main/java/ml/docilealligator/infinityforreddit/utils/Utils.java

@@ -74,12 +74,6 @@ public final class Utils {
             Pattern.compile("((?<=[\\s])|^)/[rRuU]/[\\w-]+/{0,1}"),
             Pattern.compile("((?<=[\\s])|^)[rRuU]/[\\w-]+/{0,1}"),
             Pattern.compile("\\^{2,}"),
-            Pattern.compile("(^|^ *|\\n *)#(?!($|\\s|#))"),
-            Pattern.compile("(^|^ *|\\n *)##(?!($|\\s|#))"),
-            Pattern.compile("(^|^ *|\\n *)###(?!($|\\s|#))"),
-            Pattern.compile("(^|^ *|\\n *)####(?!($|\\s|#))"),
-            Pattern.compile("(^|^ *|\\n *)#####(?!($|\\s|#))"),
-            Pattern.compile("(^|^ *|\\n *)######(?!($|\\s|#))"),
             Pattern.compile("!\\[gif]\\(giphy\\|\\w+\\)"),
             Pattern.compile("!\\[gif]\\(giphy\\|\\w+\\|downsized\\)"),
             Pattern.compile("!\\[gif]\\(emote\\|\\w+\\|\\w+\\)"),
@@ -89,12 +83,6 @@ public final class Utils {
         String regexed = REGEX_PATTERNS[0].matcher(markdown).replaceAll("[$0](https://www.reddit.com$0)");
         regexed = REGEX_PATTERNS[1].matcher(regexed).replaceAll("[$0](https://www.reddit.com/$0)");
         regexed = REGEX_PATTERNS[2].matcher(regexed).replaceAll("^");
-        regexed = REGEX_PATTERNS[3].matcher(regexed).replaceAll("$0 ");
-        regexed = REGEX_PATTERNS[4].matcher(regexed).replaceAll("$0 ");
-        regexed = REGEX_PATTERNS[5].matcher(regexed).replaceAll("$0 ");
-        regexed = REGEX_PATTERNS[6].matcher(regexed).replaceAll("$0 ");
-        regexed = REGEX_PATTERNS[7].matcher(regexed).replaceAll("$0 ");
-        regexed = REGEX_PATTERNS[8].matcher(regexed).replaceAll("$0 ");
 
         //return fixSuperScript(regexed);
         // We don't want to fix super scripts here because we need the original markdown later for editing posts
@@ -161,21 +149,21 @@ public final class Utils {
 
     public static String parseInlineGifInComments(String markdown) {
         StringBuilder markdownStringBuilder = new StringBuilder(markdown);
-        Pattern inlineGifPattern = REGEX_PATTERNS[9];
+        Pattern inlineGifPattern = REGEX_PATTERNS[3];
         Matcher matcher = inlineGifPattern.matcher(markdownStringBuilder);
         while (matcher.find()) {
             markdownStringBuilder.replace(matcher.start(), matcher.end(), "[gif](https://i.giphy.com/media/" + markdownStringBuilder.substring(matcher.start() + "![gif](giphy|".length(), matcher.end() - 1) + "/giphy.mp4)");
             matcher = inlineGifPattern.matcher(markdownStringBuilder);
         }
 
-        Pattern inlineGifPattern2 = REGEX_PATTERNS[10];
+        Pattern inlineGifPattern2 = REGEX_PATTERNS[4];
         Matcher matcher2 = inlineGifPattern2.matcher(markdownStringBuilder);
         while (matcher2.find()) {
             markdownStringBuilder.replace(matcher2.start(), matcher2.end(), "[gif](https://i.giphy.com/media/" + markdownStringBuilder.substring(matcher2.start() + "![gif](giphy|".length(), matcher2.end() - "|downsized\\)".length() + 1) + "/giphy.mp4)");
             matcher2 = inlineGifPattern2.matcher(markdownStringBuilder);
         }
 
-        Pattern inlineGifPattern3 = REGEX_PATTERNS[11];
+        Pattern inlineGifPattern3 = REGEX_PATTERNS[5];
         Matcher matcher3 = inlineGifPattern3.matcher(markdownStringBuilder);
         while (matcher3.find()) {
             markdownStringBuilder.replace(matcher3.start(), matcher3.end(),