Add Reveddit fallback for retrieving deleted comments (#591)

app/src/main/java/ml/docilealligator/infinityforreddit/AppModule.java

@@ -146,6 +146,16 @@ class AppModule {
                 .build();
     }
 
+    @Provides
+    @Named("reveddit")
+    @Singleton
+    Retrofit provideRevedditRetrofit() {
+        return new Retrofit.Builder()
+                .baseUrl(APIUtils.REVEDDIT_API_BASE_URI)
+                .addConverterFactory(ScalarsConverterFactory.create())
+                .build();
+    }
+
     @Provides
     @Named("vReddIt")
     @Singleton

app/src/main/java/ml/docilealligator/infinityforreddit/apis/PushshiftAPI.java

@@ -10,4 +10,13 @@ public interface PushshiftAPI {
 
     @GET("reddit/submission/search/")
     Call<String> getRemovedPost(@Query("ids") String postId);
+
+    @GET("reddit/comment/search/")
+    Call<String> searchComments(@Query("link_id") String linkId,
+                                @Query("limit") int limit,
+                                @Query("sort") String sort,
+                                @Query(value = "fields", encoded = true) String fields,
+                                @Query("after") long after,
+                                @Query("before") long before,
+                                @Query("q") String query);
 }

app/src/main/java/ml/docilealligator/infinityforreddit/apis/RevedditAPI.java

@@ -0,0 +1,20 @@
+package ml.docilealligator.infinityforreddit.apis;
+
+import java.util.Map;
+
+import retrofit2.Call;
+import retrofit2.http.GET;
+import retrofit2.http.HeaderMap;
+import retrofit2.http.Query;
+
+public interface RevedditAPI {
+    @GET("/short/thread-comments/")
+    Call<String> getRemovedComments(@HeaderMap Map<String, String> headers,
+                                    @Query("link_id") String threadId,
+                                    @Query("after") long after,
+                                    @Query("root_comment_id") String rootCommentId,
+                                    @Query("comment_id") String commentId,
+                                    @Query("num_comments") int numComments,
+                                    @Query("post_created_utc") long postCreatedUtc,
+                                    @Query("focus_comment_removed") boolean focusCommentRemoved);
+}

app/src/main/java/ml/docilealligator/infinityforreddit/comment/FetchRemovedComment.java

@@ -4,6 +4,7 @@ import android.os.AsyncTask;
 
 import androidx.annotation.NonNull;
 
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 
@@ -37,14 +38,45 @@ public class FetchRemovedComment {
                 });
     }
 
-    private static Comment parseRemovedComment(JSONObject result, Comment comment) throws JSONException {
+    // At the moment of writing this code, directly fetching a removed comment from
+    // Pushift.io API returns an Internal Server Error, so we temporarily do it this way instead.
+    // If this fails to return the removed comment, we try our luck with Reveddit API
+    public static void searchRemovedComment(Retrofit retrofit, Comment comment, FetchRemovedCommentListener listener) {
+        long after = (comment.getCommentTimeMillis() / 1000) - 1; // 1 second before comment creation epoch
+        retrofit.create(PushshiftAPI.class).searchComments(
+                comment.getLinkId(),
+                3000,
+                "asc",
+                "id,author,body",
+                after,
+                after + 43200, // 12 Hours later
+                "*")
+                .enqueue(new Callback<String>() {
+                    @Override
+                    public void onResponse(@NonNull Call<String> call, @NonNull Response<String> response) {
+                        if (response.isSuccessful()) {
+                            new ParseCommentAsyncTask(response.body(), comment, listener).execute();
+                        } else {
+                            listener.fetchFailed();
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(@NonNull Call<String> call, @NonNull Throwable t) {
+                        t.printStackTrace();
+                        listener.fetchFailed();
+                    }
+                });
+    }
+
+    private static Comment parseRemovedComment(@NonNull JSONObject result, Comment comment) throws JSONException {
         String id = result.getString(JSONUtils.ID_KEY);
         String author = result.getString(JSONUtils.AUTHOR_KEY);
         String body = Utils.modifyMarkdown(result.optString(JSONUtils.BODY_KEY).trim());
 
-        if ( id.equals(comment.getId()) &&
+        if (id.equals(comment.getId()) &&
-           (!author.equals(comment.getAuthor()) ||
+                (!author.equals(comment.getAuthor()) ||
-            !body.equals(comment.getCommentRawText()))
+                        !body.equals(comment.getCommentRawText()))
         ) {
             comment.setAuthor(author);
             comment.setCommentMarkdown(body);
@@ -63,8 +95,8 @@ public class FetchRemovedComment {
 
     private static class ParseCommentAsyncTask extends AsyncTask<Void, Void, Void> {
 
-        private String responseBody;
+        private final String responseBody;
-        private FetchRemovedCommentListener listener;
+        private final FetchRemovedCommentListener listener;
         Comment comment;
 
         public ParseCommentAsyncTask(String responseBody, Comment comment, FetchRemovedCommentListener listener) {
@@ -76,8 +108,17 @@ public class FetchRemovedComment {
         @Override
         protected Void doInBackground(Void... voids) {
             try {
-                JSONObject commentJSON = new JSONObject(responseBody).getJSONArray(JSONUtils.DATA_KEY).getJSONObject(0);
+                JSONArray commentJSONArray = new JSONObject(responseBody).getJSONArray(JSONUtils.DATA_KEY);
-                comment = parseRemovedComment(commentJSON, comment);
+                JSONObject commentFound = null;
+                for (int i = 0; i < commentJSONArray.length(); i++) {
+                    JSONObject commentJSON = commentJSONArray.getJSONObject(i);
+                    if (!commentJSON.isNull("id") && commentJSON.getString("id").equals(comment.getId())) {
+                        commentFound = commentJSON;
+                        break;
+                    }
+                }
+                assert commentFound != null;
+                comment = parseRemovedComment(commentFound, comment);
             } catch (JSONException e) {
                 e.printStackTrace();
                 comment = null;

app/src/main/java/ml/docilealligator/infinityforreddit/comment/FetchRemovedCommentReveddit.java

@@ -0,0 +1,108 @@
+package ml.docilealligator.infinityforreddit.comment;
+
+import android.os.AsyncTask;
+
+import androidx.annotation.NonNull;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import ml.docilealligator.infinityforreddit.apis.RevedditAPI;
+import ml.docilealligator.infinityforreddit.utils.APIUtils;
+import ml.docilealligator.infinityforreddit.utils.JSONUtils;
+import ml.docilealligator.infinityforreddit.utils.Utils;
+import retrofit2.Call;
+import retrofit2.Callback;
+import retrofit2.Response;
+import retrofit2.Retrofit;
+
+public class FetchRemovedCommentReveddit {
+
+    public static void fetchRemovedComment(Retrofit retrofit, Comment comment, long postCreatedUtc, int nComments, FetchRemovedCommentListener listener) {
+        String parentIdWithoutPrefix = comment.getParentId().substring(3);
+        String rootCommentId = parentIdWithoutPrefix.equals(comment.getLinkId()) ? comment.getId() : parentIdWithoutPrefix;
+        retrofit.create(RevedditAPI.class).getRemovedComments(
+                APIUtils.getRevedditHeader(),
+                comment.getLinkId(),
+                (comment.getCommentTimeMillis() / 1000) - 1,
+                rootCommentId,
+                comment.getId(),
+                nComments,
+                postCreatedUtc / 1000,
+                true)
+                .enqueue(new Callback<String>() {
+                    @Override
+                    public void onResponse(@NonNull Call<String> call, @NonNull Response<String> response) {
+                        if (response.isSuccessful()) {
+                            new ParseCommentAsyncTask(response.body(), comment, listener).execute();
+                        } else {
+                            listener.fetchFailed();
+                        }
+                    }
+
+                    @Override
+                    public void onFailure(@NonNull Call<String> call, @NonNull Throwable t) {
+                        t.printStackTrace();
+                        listener.fetchFailed();
+                    }
+                });
+    }
+
+    private static Comment parseRemovedComment(JSONObject result, Comment comment) throws JSONException {
+        String id = result.getString(JSONUtils.ID_KEY);
+        String author = result.getString(JSONUtils.AUTHOR_KEY);
+        String body = Utils.modifyMarkdown(result.optString(JSONUtils.BODY_KEY).trim());
+
+        if (id.equals(comment.getId()) &&
+                (!author.equals(comment.getAuthor()) ||
+                        !body.equals(comment.getCommentRawText()))
+        ) {
+            comment.setAuthor(author);
+            comment.setCommentMarkdown(body);
+            comment.setCommentRawText(body);
+            return comment;
+        } else {
+            return null;
+        }
+    }
+
+    public interface FetchRemovedCommentListener {
+        void fetchSuccess(Comment comment);
+
+        void fetchFailed();
+    }
+
+    private static class ParseCommentAsyncTask extends AsyncTask<Void, Void, Void> {
+
+        private final String responseBody;
+        private final FetchRemovedCommentListener listener;
+        Comment comment;
+
+        public ParseCommentAsyncTask(String responseBody, Comment comment, FetchRemovedCommentListener listener) {
+            this.responseBody = responseBody;
+            this.comment = comment;
+            this.listener = listener;
+        }
+
+        @Override
+        protected Void doInBackground(Void... voids) {
+            try {
+                JSONObject commentJSON = new JSONObject(responseBody).getJSONObject(comment.getId());
+                comment = parseRemovedComment(commentJSON, comment);
+            } catch (JSONException e) {
+                e.printStackTrace();
+                comment = null;
+            }
+            return null;
+        }
+
+        @Override
+        protected void onPostExecute(Void aVoid) {
+            super.onPostExecute(aVoid);
+            if (comment != null)
+                listener.fetchSuccess(comment);
+            else
+                listener.fetchFailed();
+        }
+    }
+}

app/src/main/java/ml/docilealligator/infinityforreddit/fragments/ViewPostDetailFragment.java

@@ -89,6 +89,7 @@ import ml.docilealligator.infinityforreddit.bottomsheetfragments.PostCommentSort
 import ml.docilealligator.infinityforreddit.comment.Comment;
 import ml.docilealligator.infinityforreddit.comment.FetchComment;
 import ml.docilealligator.infinityforreddit.comment.FetchRemovedComment;
+import ml.docilealligator.infinityforreddit.comment.FetchRemovedCommentReveddit;
 import ml.docilealligator.infinityforreddit.comment.ParseComment;
 import ml.docilealligator.infinityforreddit.customtheme.CustomThemeWrapper;
 import ml.docilealligator.infinityforreddit.customviews.CustomToroContainer;
@@ -144,6 +145,9 @@ public class ViewPostDetailFragment extends Fragment implements FragmentCommunic
     @Named("pushshift")
     Retrofit pushshiftRetrofit;
     @Inject
+    @Named("reveddit")
+    Retrofit revedditRetrofit;
+    @Inject
     @Named("oauth")
     Retrofit mOauthRetrofit;
     @Inject
@@ -1717,7 +1721,7 @@ public class ViewPostDetailFragment extends Fragment implements FragmentCommunic
 
     public void showRemovedComment(Comment comment, int position) {
         Toast.makeText(activity, R.string.fetching_removed_comment, Toast.LENGTH_SHORT).show();
-        FetchRemovedComment.fetchRemovedComment(
+        FetchRemovedComment.searchRemovedComment(
                 pushshiftRetrofit,
                 comment,
                 new FetchRemovedComment.FetchRemovedCommentListener() {
@@ -1728,7 +1732,18 @@ public class ViewPostDetailFragment extends Fragment implements FragmentCommunic
 
                     @Override
                     public void fetchFailed() {
-                        Toast.makeText(activity, R.string.show_removed_comment_failed, Toast.LENGTH_SHORT).show();
+                        // Reveddit fallback
+                        FetchRemovedCommentReveddit.fetchRemovedComment(revedditRetrofit, comment, mPost.getPostTimeMillis(), mPost.getNComments(), new FetchRemovedCommentReveddit.FetchRemovedCommentListener() {
+                            @Override
+                            public void fetchSuccess(Comment comment) {
+                                mCommentsAdapter.editComment(comment.getAuthor(), comment.getCommentMarkdown(), position);
+                            }
+
+                            @Override
+                            public void fetchFailed() {
+                                Toast.makeText(activity, R.string.show_removed_comment_failed, Toast.LENGTH_SHORT).show();
+                            }
+                        });
                     }
                 });
     }

app/src/main/java/ml/docilealligator/infinityforreddit/post/ParsePost.java

@@ -156,8 +156,10 @@ public class ParsePost {
         boolean nsfw = data.getBoolean(JSONUtils.NSFW_KEY);
         boolean stickied = data.getBoolean(JSONUtils.STICKIED_KEY);
         boolean archived = data.getBoolean(JSONUtils.ARCHIVED_KEY);
-        boolean locked = data.getBoolean(JSONUtils.LOCKEC_KEY);
+        boolean locked = data.getBoolean(JSONUtils.LOCKED_KEY);
         boolean saved = data.getBoolean(JSONUtils.SAVED_KEY);
+        boolean deleted = !data.isNull(JSONUtils.REMOVED_BY_CATEGORY_KEY) && data.getString(JSONUtils.REMOVED_BY_CATEGORY_KEY).equals("deleted");
+        boolean removed = !data.isNull(JSONUtils.REMOVED_BY_CATEGORY_KEY) && data.getString(JSONUtils.REMOVED_BY_CATEGORY_KEY).equals("moderator");
         StringBuilder postFlairHTMLBuilder = new StringBuilder();
         String flair = "";
         if (data.has(JSONUtils.LINK_FLAIR_RICHTEXT_KEY)) {
@@ -231,7 +233,7 @@ public class ParsePost {
                     author, authorFlair, authorFlairHTMLBuilder.toString(),
                     postTime, title, previews,
                     score, voteType, nComments, upvoteRatio, flair, awardingsBuilder.toString(), nAwards, hidden,
-                    spoiler, nsfw, stickied, archived, locked, saved, true);
+                    spoiler, nsfw, stickied, archived, locked, saved, deleted, removed, true);
             post.setCrosspostParentId(crosspostParent.getId());
             return post;
         } else {
@@ -239,7 +241,7 @@ public class ParsePost {
                     author, authorFlair, authorFlairHTMLBuilder.toString(),
                     postTime, title, previews,
                     score, voteType, nComments, upvoteRatio, flair, awardingsBuilder.toString(), nAwards, hidden,
-                    spoiler, nsfw, stickied, archived, locked, saved, false);
+                    spoiler, nsfw, stickied, archived, locked, saved, deleted, removed, false);
         }
     }
 
@@ -250,7 +252,7 @@ public class ParsePost {
                                   int score, int voteType, int nComments, int upvoteRatio, String flair,
                                   String awards, int nAwards, boolean hidden, boolean spoiler,
                                   boolean nsfw, boolean stickied, boolean archived, boolean locked,
-                                  boolean saved, boolean isCrosspost) throws JSONException {
+                                  boolean saved, boolean deleted, boolean removed, boolean isCrosspost) throws JSONException {
         Post post;
 
         boolean isVideo = data.getBoolean(JSONUtils.IS_VIDEO_KEY);

app/src/main/java/ml/docilealligator/infinityforreddit/utils/APIUtils.java

@@ -22,6 +22,7 @@ public class APIUtils {
     public static final String REDGIFS_API_BASE_URI = "https://api.redgifs.com/v1/gfycats/";
     public static final String IMGUR_API_BASE_URI = "https://api.imgur.com/3/";
     public static final String PUSHSHIFT_API_BASE_URI = "https://api.pushshift.io/";
+    public static final String REVEDDIT_API_BASE_URI = "https://api.reveddit.com/";
     public static final String STRAPI_BASE_URI = "https://strapi.reddit.com";
     public static final String STREAMABLE_API_BASE_URI = "https://api.streamable.com";
 
@@ -105,6 +106,11 @@ public class APIUtils {
     public static final String GILD_TYPE = "gild_type";
     public static final String IS_ANONYMOUS = "is_anonymous";
 
+    public static final String ORIGIN_KEY = "Origin";
+    public static final String REVEDDIT_ORIGIN = "https://www.reveddit.com";
+    public static final String REFERER_KEY = "Referer";
+    public static final String REVEDDIT_REFERER = "https://www.reveddit.com/";
+
     public static Map<String, String> getHttpBasicAuthHeader() {
         Map<String, String> params = new HashMap<>();
         String credentials = String.format("%s:%s", APIUtils.CLIENT_ID, "");
@@ -123,4 +129,12 @@ public class APIUtils {
     public static RequestBody getRequestBody(String s) {
         return RequestBody.create(s, MediaType.parse("text/plain"));
     }
+
+    public static Map<String, String> getRevedditHeader() {
+        Map<String, String> params = new HashMap<>();
+        params.put(APIUtils.ORIGIN_KEY, APIUtils.REVEDDIT_ORIGIN);
+        params.put(APIUtils.REFERER_KEY, APIUtils.REVEDDIT_REFERER);
+        params.put(APIUtils.USER_AGENT_KEY, APIUtils.USER_AGENT);
+        return params;
+    }
 }

app/src/main/java/ml/docilealligator/infinityforreddit/utils/JSONUtils.java

@@ -85,8 +85,10 @@ public class JSONUtils {
     public static final String DESCRIPTION_HTML_KEY = "description_html";
     public static final String DESCRIPTION_MD_KEY = "description_md";
     public static final String ARCHIVED_KEY = "archived";
-    public static final String LOCKEC_KEY = "locked";
+    public static final String LOCKED_KEY = "locked";
     public static final String SAVED_KEY = "saved";
+    public static final String REMOVED_KEY = "removed";
+    public static final String REMOVED_BY_CATEGORY_KEY = "removed_by_category";
     public static final String TEXT_EDITABLE_KEY = "text_editable";
     public static final String SUBJECT_KEY = "subject";
     public static final String CONTEXT_KEY = "context";