From 421342734c5159e84f29c828e09752fe051c2f58 Mon Sep 17 00:00:00 2001 From: Balazs Toldi Date: Fri, 10 Nov 2023 22:12:42 +0100 Subject: [PATCH] Multicommunity Working? This commit improves the existing PoC multicommunity implementation with a "less hacked together" one --- .../activities/CreateMultiRedditActivity.java | 2 +- .../post/PostPagingSource.java | 87 ++++++--- .../infinityforlemmy/post/PostViewModel.java | 15 +- .../MultiCommunityPostPagingSource.kt | 181 ++++++++++++++++++ .../utils/MultiCommunityUtils.kt | 27 +++ 5 files changed, 281 insertions(+), 31 deletions(-) create mode 100644 app/src/main/kotlin/eu/toldi/infinityforlemmy/multicommunity/MultiCommunityPostPagingSource.kt create mode 100644 app/src/main/kotlin/eu/toldi/infinityforlemmy/utils/MultiCommunityUtils.kt diff --git a/app/src/main/java/eu/toldi/infinityforlemmy/activities/CreateMultiRedditActivity.java b/app/src/main/java/eu/toldi/infinityforlemmy/activities/CreateMultiRedditActivity.java index fedf458c..b6155fd6 100644 --- a/app/src/main/java/eu/toldi/infinityforlemmy/activities/CreateMultiRedditActivity.java +++ b/app/src/main/java/eu/toldi/infinityforlemmy/activities/CreateMultiRedditActivity.java @@ -114,7 +114,7 @@ public class CreateMultiRedditActivity extends BaseActivity { getSupportActionBar().setDisplayHomeAsUpEnabled(true); mAccessToken = mCurrentAccountSharedPreferences.getString(SharedPreferencesUtils.ACCESS_TOKEN, null); - mAccountName = mCurrentAccountSharedPreferences.getString(SharedPreferencesUtils.ACCOUNT_NAME, "-"); + mAccountName = mCurrentAccountSharedPreferences.getString(SharedPreferencesUtils.ACCOUNT_QUALIFIED_NAME, "-"); visibilityLinearLayout.setVisibility(View.GONE); if (mAccessToken == null) { diff --git a/app/src/main/java/eu/toldi/infinityforlemmy/post/PostPagingSource.java b/app/src/main/java/eu/toldi/infinityforlemmy/post/PostPagingSource.java index 54a4ddc1..f882264b 100644 --- a/app/src/main/java/eu/toldi/infinityforlemmy/post/PostPagingSource.java +++ b/app/src/main/java/eu/toldi/infinityforlemmy/post/PostPagingSource.java @@ -1,7 +1,6 @@ package eu.toldi.infinityforlemmy.post; import android.content.SharedPreferences; -import android.os.Build; import androidx.annotation.NonNull; import androidx.paging.ListenableFuturePagingSource; @@ -15,8 +14,10 @@ import org.jetbrains.annotations.Nullable; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.concurrent.Executor; import java.util.regex.Pattern; @@ -26,6 +27,7 @@ import eu.toldi.infinityforlemmy.SortType; import eu.toldi.infinityforlemmy.apis.LemmyAPI; import eu.toldi.infinityforlemmy.post.enrich.PostEnricher; import eu.toldi.infinityforlemmy.postfilter.PostFilter; +import eu.toldi.infinityforlemmy.utils.MultiCommunityUtils; import eu.toldi.infinityforlemmy.utils.SharedPreferencesUtils; import retrofit2.HttpException; import retrofit2.Response; @@ -226,7 +228,7 @@ public class PostPagingSource extends ListenableFuturePagingSource> undisplayedPosts = new HashMap<>(); + private ListenableFuture> loadMultipleSubredditPosts(@NonNull LoadParams loadParams, LemmyAPI api, List communities) { List>> futures = new ArrayList<>(); + List combinedPostsFromCache = new ArrayList<>(); + // Check and use undisplayed posts first for (String community : communities) { - ListenableFuture> subredditPost; - - subredditPost = api.getPostsListenableFuture(null, sortType.getType().value, loadParams.getKey(), 25, null, community, false, accessToken); - - ListenableFuture> communityFuture = Futures.transform(subredditPost, - this::transformData, executor); - - ListenableFuture> partialLoadResultFuture = - Futures.catching(communityFuture, HttpException.class, - LoadResult.Error::new, executor); - - futures.add(Futures.catching(partialLoadResultFuture, - IOException.class, LoadResult.Error::new, executor)); + futures.add(fetchPostsFromCommunity(api, loadParams, community)); } + // Combine and sort posts from cache and fetched return Futures.transform(Futures.successfulAsList(futures), results -> { - List combinedPosts = new ArrayList<>(); for (LoadResult result : results) { if (result instanceof LoadResult.Page) { - combinedPosts.addAll(((LoadResult.Page) result).getData()); - } else if (result instanceof LoadResult.Error) { - // Handle or propagate the error if needed - return result; + combinedPostsFromCache.addAll(((LoadResult.Page) result).getData()); + } + } + switch (sortType.getType()) { + default: + case NEW: + MultiCommunityUtils.INSTANCE.sortByNewest(combinedPostsFromCache); + case TOP_ALL: + case TOP_YEAR: + case TOP_NINE_MONTHS: + case TOP_SIX_MONTHS: + case TOP_THREE_MONTHS: + case TOP_MONTH: + case TOP_WEEK: + case TOP_DAY: + case TOP_TWELVE_HOURS: + case TOP_SIX_HOURS: + case TOP_HOUR: + MultiCommunityUtils.INSTANCE.sortByScore(combinedPostsFromCache); + + } + + List result = MultiCommunityUtils.INSTANCE.takeFirstN(combinedPostsFromCache, 25); + // Gather undisplayed posts + for (int i = result.size(); i < combinedPostsFromCache.size(); i++) { + Post post = combinedPostsFromCache.get(i); + if (undisplayedPosts.containsKey(post.getCommunityInfo().getQualifiedName())) { + undisplayedPosts.get(post.getCommunityInfo().getQualifiedName()).add(post); + } else { + undisplayedPosts.put(post.getCommunityInfo().getQualifiedName(), new ArrayList<>(Arrays.asList(post))); } } - if (sortType.getType().equals(SortType.Type.NEW)) { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { - combinedPosts.sort((o1, o2) -> Long.compare(o2.getPostTimeMillis(), o1.getPostTimeMillis())); - } - } - return new LoadResult.Page<>(combinedPosts, null, null); + Integer prevKey = result.size() > 0 ? (loadParams.getKey() != null) ? loadParams.getKey() : 1 : null; + Integer nextKey = (prevKey != null) ? prevKey + 1 : null; + + return new LoadResult.Page<>(result, prevKey, nextKey); }, executor); } + private ListenableFuture> fetchPostsFromCommunity(LemmyAPI api, LoadParams loadParams, String community) { + + ListenableFuture> subredditPost; + + subredditPost = api.getPostsListenableFuture(null, sortType.getType().value, loadParams.getKey(), 25, null, community, false, accessToken); + + ListenableFuture> communityFuture = Futures.transform(subredditPost, + this::transformData, executor); + + ListenableFuture> partialLoadResultFuture = + Futures.catching(communityFuture, HttpException.class, + LoadResult.Error::new, executor); + + return Futures.catching(partialLoadResultFuture, + IOException.class, LoadResult.Error::new, executor); + } + /* private ListenableFuture> loadMultiRedditPosts(@NonNull LoadParams loadParams, LemmyAPI api) { diff --git a/app/src/main/java/eu/toldi/infinityforlemmy/post/PostViewModel.java b/app/src/main/java/eu/toldi/infinityforlemmy/post/PostViewModel.java index b692c0f4..0bb85d90 100644 --- a/app/src/main/java/eu/toldi/infinityforlemmy/post/PostViewModel.java +++ b/app/src/main/java/eu/toldi/infinityforlemmy/post/PostViewModel.java @@ -17,12 +17,16 @@ import androidx.paging.PagingConfig; import androidx.paging.PagingData; import androidx.paging.PagingDataTransforms; import androidx.paging.PagingLiveData; +import androidx.paging.PagingSource; import java.util.List; import java.util.concurrent.Executor; +import java.util.regex.Pattern; import eu.toldi.infinityforlemmy.RetrofitHolder; import eu.toldi.infinityforlemmy.SortType; +import eu.toldi.infinityforlemmy.apis.LemmyAPI; +import eu.toldi.infinityforlemmy.multicommunity.MulticommunityPagingSource; import eu.toldi.infinityforlemmy.post.enrich.PostEnricher; import eu.toldi.infinityforlemmy.postfilter.PostFilter; import eu.toldi.infinityforlemmy.utils.SharedPreferencesUtils; @@ -230,8 +234,8 @@ public class PostViewModel extends ViewModel { currentlyReadPostIdsLiveData.setValue(true); } - public PostPagingSource returnPagingSoruce() { - PostPagingSource paging3PagingSource; + public PagingSource returnPagingSoruce() { + PagingSource paging3PagingSource; switch (postType) { case PostPagingSource.TYPE_FRONT_PAGE: paging3PagingSource = new PostPagingSource(executor, retrofit, accessToken, accountName, @@ -239,12 +243,15 @@ public class PostViewModel extends ViewModel { postFilter, readPostList, name, postEnricher); break; case PostPagingSource.TYPE_SUBREDDIT: - case PostPagingSource.TYPE_MULTI_REDDIT: - case PostPagingSource.TYPE_ANONYMOUS_FRONT_PAGE: paging3PagingSource = new PostPagingSource(executor, retrofit, accessToken, accountName, sharedPreferences, postFeedScrolledPositionSharedPreferences, name, postType, sortType, postFilter, readPostList, postEnricher); break; + case PostPagingSource.TYPE_MULTI_REDDIT: + case PostPagingSource.TYPE_ANONYMOUS_FRONT_PAGE: + paging3PagingSource = new MulticommunityPagingSource(retrofit.getRetrofit().create(LemmyAPI.class), List.of(name.split(Pattern.quote(","))), accessToken, + sortType, executor, postFilter, readPostList, postEnricher); + break; case PostPagingSource.TYPE_SEARCH: paging3PagingSource = new PostPagingSource(executor, retrofit, accessToken, accountName, sharedPreferences, postFeedScrolledPositionSharedPreferences, name, query, trendingSource, diff --git a/app/src/main/kotlin/eu/toldi/infinityforlemmy/multicommunity/MultiCommunityPostPagingSource.kt b/app/src/main/kotlin/eu/toldi/infinityforlemmy/multicommunity/MultiCommunityPostPagingSource.kt new file mode 100644 index 00000000..d33cfdd2 --- /dev/null +++ b/app/src/main/kotlin/eu/toldi/infinityforlemmy/multicommunity/MultiCommunityPostPagingSource.kt @@ -0,0 +1,181 @@ +package eu.toldi.infinityforlemmy.multicommunity + +import androidx.paging.ListenableFuturePagingSource +import androidx.paging.PagingState +import com.google.common.base.Function +import com.google.common.util.concurrent.Futures +import com.google.common.util.concurrent.ListenableFuture +import eu.toldi.infinityforlemmy.SortType +import eu.toldi.infinityforlemmy.apis.LemmyAPI +import eu.toldi.infinityforlemmy.post.ParsePost +import eu.toldi.infinityforlemmy.post.Post +import eu.toldi.infinityforlemmy.post.enrich.PostEnricher +import eu.toldi.infinityforlemmy.postfilter.PostFilter +import eu.toldi.infinityforlemmy.utils.MultiCommunityUtils.sortByNewest +import eu.toldi.infinityforlemmy.utils.MultiCommunityUtils.sortByScore +import retrofit2.HttpException +import retrofit2.Response +import java.io.IOException +import java.util.concurrent.Executor + + +class MulticommunityPagingSource( + private val api: LemmyAPI, + private val communities: List, + private val accessToken: String, + private val sortType: SortType, + private val executor: Executor, + private val postFilter: PostFilter, + private val readPostList: List, private val postEnricher: PostEnricher +) : ListenableFuturePagingSource, Post>() { + + private val postLinkedHashSet: LinkedHashSet = LinkedHashSet() + private val undisplayedPosts = mutableMapOf>() + + override fun loadFuture(loadParams: LoadParams>): ListenableFuture, Post>> { + val currentPageMap = loadParams.key ?: communities.associateWith { 1 } + + // Initialize a list to hold futures of post loading results + val futuresList = mutableListOf>>() + val combinedPosts = mutableListOf() + val wasCached = mutableMapOf() + // Loop through each community and fetch posts + for ((community, pageNumber) in currentPageMap) { + + if (undisplayedPosts.containsKey(community) && undisplayedPosts[community]!!.size > 10) { + val posts: List = undisplayedPosts[community] ?: listOf() + combinedPosts.addAll(posts) + undisplayedPosts[community]!!.clear() // Clear used posts + wasCached[community] = true + } else { + val future = fetchPostsFromCommunity(api, pageNumber, community) + futuresList.add(future) + wasCached[community] = false + } + } + + val combinedFuture: ListenableFuture, Post>> = + Futures.transform(Futures.successfulAsList(futuresList), { individualResults -> + // Combine the results from individual communities + + val nextPageMap = mutableMapOf() + + for ((index, result) in individualResults.withIndex()) { + if (result is LoadResult.Page) { + combinedPosts.addAll(result.data) + val addition = if (wasCached[communities[index]] == true) 0 else 1 + nextPageMap[communities[index]] = + result.nextKey ?: (currentPageMap[communities[index]]?.plus(addition)) + ?: 1 + } + // Handle other cases like LoadResult.Error + } + + val sorted = when (sortType.type) { + SortType.Type.NEW -> { + sortByNewest(combinedPosts) + } + + SortType.Type.TOP_ALL, SortType.Type.TOP_YEAR, SortType.Type.TOP_NINE_MONTHS, SortType.Type.TOP_SIX_MONTHS, SortType.Type.TOP_THREE_MONTHS, SortType.Type.TOP_MONTH, SortType.Type.TOP_WEEK, SortType.Type.TOP_DAY, SortType.Type.TOP_TWELVE_HOURS, SortType.Type.TOP_SIX_HOURS, SortType.Type.TOP_HOUR + -> sortByScore(combinedPosts) + + else -> { + sortByNewest(combinedPosts) + } + } + + val filteredPosts = sorted.take(25) + + // Store undisplayed posts + for (post in combinedPosts) { + if (!filteredPosts.contains(post)) { + if (undisplayedPosts.containsKey(post.subredditNamePrefixed)) { + undisplayedPosts[post.subredditNamePrefixed]?.add(post) + } else { + undisplayedPosts[post.subredditNamePrefixed] = mutableListOf(post) + } + } + } + + LoadResult.Page( + data = filteredPosts, + prevKey = null, // Define prevKey logic if needed + nextKey = nextPageMap + ) + }, executor) + val partialLoadResultFuture = + Futures.catching, Post>, HttpException>( + combinedFuture, + HttpException::class.java, + Function, Post>> { throwable: HttpException -> + LoadResult.Error(throwable) + }, executor + ) + + return partialLoadResultFuture + + } + + private fun fetchPostsFromCommunity( + api: LemmyAPI, + pageNumber: Int, + community: String + ): ListenableFuture> { + val subredditPost: ListenableFuture> + subredditPost = api.getPostsListenableFuture( + null, + sortType.getType().value, + pageNumber, + 25, + null, + community, + false, + accessToken + ) + val communityFuture: ListenableFuture> = Futures.transform( + subredditPost, + { response -> transformData(response, pageNumber) }, executor + ) + + val partialLoadResultFuture: ListenableFuture> = + Futures.catching( + communityFuture, + HttpException::class.java, + { throwable -> LoadResult.Error(throwable) }, + executor + ) + + return Futures.catching( + partialLoadResultFuture, + IOException::class.java, + Function> { throwable -> + LoadResult.Error(throwable) + }, executor + ) + } + + + fun transformData(response: Response, pageNumber: Int): LoadResult { + if (!response.isSuccessful) return LoadResult.Error(Exception("Response failed")) + + val responseString = + response.body() ?: return LoadResult.Error(Exception("Empty response body")) + val newPosts = + ParsePost.parsePostsSync(responseString, -1, postFilter, readPostList, postEnricher) + ?: return LoadResult.Error(Exception("Error parsing posts")) + + // Filter out already linked posts + val trulyNewPosts = newPosts.filterNot(postLinkedHashSet::contains) + postLinkedHashSet.addAll(trulyNewPosts) + + val nextKey = if (trulyNewPosts.isNotEmpty()) pageNumber + 1 else null + return LoadResult.Page(trulyNewPosts, null, nextKey) + } + + + override fun getRefreshKey(pagingState: PagingState, Post>): Map? { + return null + } + + // Additional methods and logic... +} diff --git a/app/src/main/kotlin/eu/toldi/infinityforlemmy/utils/MultiCommunityUtils.kt b/app/src/main/kotlin/eu/toldi/infinityforlemmy/utils/MultiCommunityUtils.kt new file mode 100644 index 00000000..a57946cc --- /dev/null +++ b/app/src/main/kotlin/eu/toldi/infinityforlemmy/utils/MultiCommunityUtils.kt @@ -0,0 +1,27 @@ +package eu.toldi.infinityforlemmy.utils + +import eu.toldi.infinityforlemmy.post.Post + +object MultiCommunityUtils { + + fun takeFirstN(posts: List, take: Int = 25): List { + return posts.take(take) + } + + fun sortByNewest(posts: List): List { + return posts.sortedByDescending { it.postTimeMillis } + } + + fun sortByOldest(posts: List): List { + return posts.sortedBy { it.postTimeMillis } + } + + fun sortByMostComments(posts: List): List { + return posts.sortedByDescending { it.nComments } + } + + fun sortByScore(posts: List): List { + return posts.sortedByDescending { it.score } + } + +} \ No newline at end of file