Improve sources' paging

app/src/main/java/eu/kanade/mangafeed/data/source/SourceManager.java

@@ -6,10 +6,10 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 
-import eu.kanade.mangafeed.data.source.online.english.Batoto;
-import eu.kanade.mangafeed.data.source.online.english.Mangahere;
-import eu.kanade.mangafeed.data.source.online.english.Mangafox;
 import eu.kanade.mangafeed.data.source.base.Source;
+import eu.kanade.mangafeed.data.source.online.english.Batoto;
+import eu.kanade.mangafeed.data.source.online.english.Mangafox;
+import eu.kanade.mangafeed.data.source.online.english.Mangahere;
 
 public class SourceManager {
 

app/src/main/java/eu/kanade/mangafeed/data/source/base/BaseSource.java

@@ -3,10 +3,13 @@ package eu.kanade.mangafeed.data.source.base;
 import com.squareup.okhttp.Headers;
 import com.squareup.okhttp.Response;
 
+import org.jsoup.nodes.Document;
+
 import java.util.List;
 
 import eu.kanade.mangafeed.data.database.models.Chapter;
 import eu.kanade.mangafeed.data.database.models.Manga;
+import eu.kanade.mangafeed.data.source.model.MangasPage;
 import rx.Observable;
 
 public abstract class BaseSource {
@@ -20,17 +23,23 @@ public abstract class BaseSource {
     // True if the source requires a login
     public abstract boolean isLoginRequired();
 
-    // Given a page number, it should return the URL of the page where the manga list is found
+    // Return the initial popular mangas URL
-    protected abstract String getUrlFromPageNumber(int page);
+    protected abstract String getInitialPopularMangasUrl();
 
-    // From the URL obtained before, this method must return a list of mangas
+    // Return the initial search url given a query
-    protected abstract List<Manga> parsePopularMangasFromHtml(String unparsedHtml);
+    protected abstract String getInitialSearchUrl(String query);
 
-    // Given a query and a page number, return the URL of the results
+    // Get the popular list of mangas from the source's parsed document
-    protected abstract String getSearchUrl(String query, int page);
+    protected abstract List<Manga> parsePopularMangasFromHtml(Document parsedHtml);
 
-    // From the URL obtained before, this method must return a list of mangas
+    // Get the next popular page URL or null if it's the last
-    protected abstract List<Manga> parseSearchFromHtml(String unparsedHtml);
+    protected abstract String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page);
+
+    // Get the searched list of mangas from the source's parsed document
+    protected abstract List<Manga> parseSearchFromHtml(Document parsedHtml);
+
+    // Get the next search page URL or null if it's the last
+    protected abstract String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query);
 
     // Given the URL of a manga and the result of the request, return the details of the manga
     protected abstract Manga parseHtmlToManga(String mangaUrl, String unparsedHtml);

app/src/main/java/eu/kanade/mangafeed/data/source/base/Source.java

@@ -1,11 +1,12 @@
 package eu.kanade.mangafeed.data.source.base;
 
-
 import android.content.Context;
 
 import com.squareup.okhttp.Headers;
 import com.squareup.okhttp.Response;
 
+import org.jsoup.Jsoup;
+
 import java.util.ArrayList;
 import java.util.List;
 
@@ -13,10 +14,11 @@ import javax.inject.Inject;
 
 import eu.kanade.mangafeed.App;
 import eu.kanade.mangafeed.data.cache.CacheManager;
-import eu.kanade.mangafeed.data.network.NetworkHelper;
-import eu.kanade.mangafeed.data.preference.PreferencesHelper;
 import eu.kanade.mangafeed.data.database.models.Chapter;
 import eu.kanade.mangafeed.data.database.models.Manga;
+import eu.kanade.mangafeed.data.network.NetworkHelper;
+import eu.kanade.mangafeed.data.preference.PreferencesHelper;
+import eu.kanade.mangafeed.data.source.model.MangasPage;
 import eu.kanade.mangafeed.data.source.model.Page;
 import rx.Observable;
 import rx.schedulers.Schedulers;
@@ -34,18 +36,29 @@ public abstract class Source extends BaseSource {
     }
 
     // Get the most popular mangas from the source
-    public Observable<List<Manga>> pullPopularMangasFromNetwork(int page) {
+    public Observable<MangasPage> pullPopularMangasFromNetwork(MangasPage page) {
-        String url = getUrlFromPageNumber(page);
+        if (page.page == 1)
+            page.url = getInitialPopularMangasUrl();
+
         return mNetworkService
-                .getStringResponse(url, mRequestHeaders, null)
+                .getStringResponse(page.url, mRequestHeaders, null)
-                .flatMap(response -> Observable.just(parsePopularMangasFromHtml(response)));
+                .map(Jsoup::parse)
+                .doOnNext(doc -> page.mangas = parsePopularMangasFromHtml(doc))
+                .doOnNext(doc -> page.nextPageUrl = parseNextPopularMangasUrl(doc, page))
+                .map(response -> page);
     }
 
     // Get mangas from the source with a query
-    public Observable<List<Manga>> searchMangasFromNetwork(String query, int page) {
+    public Observable<MangasPage> searchMangasFromNetwork(MangasPage page, String query) {
+        if (page.page == 1)
+            page.url = getInitialSearchUrl(query);
+
         return mNetworkService
-                .getStringResponse(getSearchUrl(query, page), mRequestHeaders, null)
+                .getStringResponse(page.url, mRequestHeaders, null)
-                .flatMap(response -> Observable.just(parseSearchFromHtml(response)));
+                .map(Jsoup::parse)
+                .doOnNext(doc -> page.mangas = parseSearchFromHtml(doc))
+                .doOnNext(doc -> page.nextPageUrl = parseNextSearchUrl(doc, page, query))
+                .map(response -> page);
     }
 
     // Get manga details from the source

app/src/main/java/eu/kanade/mangafeed/data/source/model/MangasPage.java

@@ -0,0 +1,18 @@
+package eu.kanade.mangafeed.data.source.model;
+
+import java.util.List;
+
+import eu.kanade.mangafeed.data.database.models.Manga;
+
+public class MangasPage {
+
+    public List<Manga> mangas;
+    public int page;
+    public String url;
+    public String nextPageUrl;
+
+    public MangasPage(int page) {
+        this.page = page;
+    }
+
+}

app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Batoto.java

@@ -25,13 +25,14 @@ import eu.kanade.mangafeed.data.source.SourceManager;
 import eu.kanade.mangafeed.data.database.models.Chapter;
 import eu.kanade.mangafeed.data.database.models.Manga;
 import eu.kanade.mangafeed.data.source.base.Source;
+import eu.kanade.mangafeed.data.source.model.MangasPage;
 import rx.Observable;
 
 public class Batoto extends Source {
 
     public static final String NAME = "Batoto (EN)";
     public static final String BASE_URL = "http://bato.to";
-    public static final String INITIAL_UPDATE_URL =
+    public static final String INITIAL_POPULAR_MANGAS_URL =
             "http://bato.to/search_ajax?order_cond=views&order=desc&p=";
     public static final String INITIAL_SEARCH_URL = "http://bato.to/search_ajax?";
     public static final String INITIAL_PAGE_URL = "http://bato.to/areader?";
@@ -112,13 +113,13 @@ public class Batoto extends Source {
     }
 
     @Override
-    protected String getUrlFromPageNumber(int page) {
+    public String getInitialPopularMangasUrl() {
-        return INITIAL_UPDATE_URL + page;
+        return INITIAL_POPULAR_MANGAS_URL + "1";
     }
 
     @Override
-    protected String getSearchUrl(String query, int page) {
+    public String getInitialSearchUrl(String query) {
-        return INITIAL_SEARCH_URL + "name=" + query + "&p=" + page;
+        return INITIAL_SEARCH_URL + "name=" + query + "&p=1";
     }
 
     @Override
@@ -141,33 +142,49 @@ public class Batoto extends Source {
         return INITIAL_PAGE_URL + "id=" + id + "&p=" + defaultPageUrl.substring(end+1);
     }
 
-    private List<Manga> parseMangasFromHtml(String unparsedHtml) {
+    @Override
-        if (unparsedHtml.contains("No (more) comics found!")) {
+    protected List<Manga> parsePopularMangasFromHtml(Document parsedHtml) {
+        if (parsedHtml.text().contains("No (more) comics found!")) {
             return new ArrayList<>();
         }
 
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
+        List<Manga> mangaList = new ArrayList<>();
 
-        List<Manga> updatedMangaList = new ArrayList<>();
+        Elements updatedHtmlBlocks = parsedHtml.select("tr:not([id]):not([class])");
-
-        Elements updatedHtmlBlocks = parsedDocument.select("tr:not([id]):not([class])");
         for (Element currentHtmlBlock : updatedHtmlBlocks) {
             Manga currentlyUpdatedManga = constructMangaFromHtmlBlock(currentHtmlBlock);
 
-            updatedMangaList.add(currentlyUpdatedManga);
+            mangaList.add(currentlyUpdatedManga);
         }
 
-        return updatedMangaList;
+        return mangaList;
     }
 
     @Override
-    public List<Manga> parsePopularMangasFromHtml(String unparsedHtml) {
+    protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) {
-        return parseMangasFromHtml(unparsedHtml);
+        Element next = parsedHtml.select("#show_more_row").first();
+        if (next == null)
+            return null;
+
+        return INITIAL_POPULAR_MANGAS_URL + (page.page + 1);
     }
 
     @Override
-    protected List<Manga> parseSearchFromHtml(String unparsedHtml) {
+    protected List<Manga> parseSearchFromHtml(Document parsedHtml) {
-        return parseMangasFromHtml(unparsedHtml);
+        if (parsedHtml.text().contains("No (more) comics found!")) {
+            return new ArrayList<>();
+        }
+
+        List<Manga> mangaList = new ArrayList<>();
+
+        Elements updatedHtmlBlocks = parsedHtml.select("tr:not([id]):not([class])");
+        for (Element currentHtmlBlock : updatedHtmlBlocks) {
+            Manga currentlyUpdatedManga = constructMangaFromHtmlBlock(currentHtmlBlock);
+
+            mangaList.add(currentlyUpdatedManga);
+        }
+
+        return mangaList;
     }
 
     private Manga constructMangaFromHtmlBlock(Element htmlBlock) {
@@ -195,6 +212,15 @@ public class Batoto extends Source {
         return mangaFromHtmlBlock;
     }
 
+    @Override
+    protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) {
+        Element next = parsedHtml.select("#show_more_row").first();
+        if (next == null)
+            return null;
+
+        return INITIAL_SEARCH_URL + "name=" + query + "&p=" + (page.page + 1);
+    }
+
     private long parseUpdateFromElement(Element updateElement) {
         String updatedDateAsString = updateElement.text();
 

app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangafox.java

@@ -14,13 +14,14 @@ import eu.kanade.mangafeed.data.source.SourceManager;
 import eu.kanade.mangafeed.data.database.models.Chapter;
 import eu.kanade.mangafeed.data.database.models.Manga;
 import eu.kanade.mangafeed.data.source.base.Source;
+import eu.kanade.mangafeed.data.source.model.MangasPage;
 
 public class Mangafox extends Source {
 
     public static final String NAME = "Mangafox (EN)";
-
+    public static final String BASE_URL = "http://mangafox.me";
-    private static final String INITIAL_UPDATE_URL = "http://mangafox.me/directory/";
+    public static final String INITIAL_POPULAR_MANGAS_URL = "http://mangafox.me/directory/";
-    private static final String INITIAL_SEARCH_URL =
+    public static final String INITIAL_SEARCH_URL =
             "http://mangafox.me/search.php?name_method=cw&advopts=1&order=az&sort=name";
 
     public Mangafox(Context context) {
@@ -43,17 +44,20 @@ public class Mangafox extends Source {
     }
 
     @Override
-    protected String getUrlFromPageNumber(int page) {
+    protected String getInitialPopularMangasUrl() {
-        return INITIAL_UPDATE_URL + page + ".htm";
+        return INITIAL_POPULAR_MANGAS_URL;
     }
 
     @Override
-    public List<Manga> parsePopularMangasFromHtml(String unparsedHtml) {
+    protected String getInitialSearchUrl(String query) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
+        return INITIAL_SEARCH_URL + "&name=" + query + "&page=1";
+    }
 
+    @Override
+    protected List<Manga> parsePopularMangasFromHtml(Document parsedHtml) {
         List<Manga> mangaList = new ArrayList<>();
 
-        Elements mangaHtmlBlocks = parsedDocument.select("div#mangalist > ul.list > li");
+        Elements mangaHtmlBlocks = parsedHtml.select("div#mangalist > ul.list > li");
         for (Element currentHtmlBlock : mangaHtmlBlocks) {
             Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock);
             mangaList.add(currentManga);
@@ -77,17 +81,19 @@ public class Mangafox extends Source {
     }
 
     @Override
-    protected String getSearchUrl(String query, int page) {
+    protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) {
-        return INITIAL_SEARCH_URL + "&name=" + query + "&page=" + page;
+        Element next = parsedHtml.select("a:has(span.next)").first();
+        if (next == null)
+            return null;
+
+        return INITIAL_POPULAR_MANGAS_URL + next.attr("href");
     }
 
     @Override
-    protected List<Manga> parseSearchFromHtml(String unparsedHtml) {
+    protected List<Manga> parseSearchFromHtml(Document parsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-
         List<Manga> mangaList = new ArrayList<>();
 
-        Elements mangaHtmlBlocks = parsedDocument.select("table#listing > tbody > tr:gt(0)");
+        Elements mangaHtmlBlocks = parsedHtml.select("table#listing > tbody > tr:gt(0)");
         for (Element currentHtmlBlock : mangaHtmlBlocks) {
             Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock);
             mangaList.add(currentManga);
@@ -110,6 +116,15 @@ public class Mangafox extends Source {
         return mangaFromHtmlBlock;
     }
 
+    @Override
+    protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) {
+        Element next = parsedHtml.select("a:has(span.next)").first();
+        if (next == null)
+            return null;
+
+        return BASE_URL + next.attr("href");
+    }
+
     @Override
     protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) {
         Document parsedDocument = Jsoup.parse(unparsedHtml);

app/src/main/java/eu/kanade/mangafeed/data/source/online/english/Mangahere.java

@@ -15,18 +15,19 @@ import java.util.Date;
 import java.util.List;
 import java.util.Locale;
 
-import eu.kanade.mangafeed.data.source.SourceManager;
 import eu.kanade.mangafeed.data.database.models.Chapter;
 import eu.kanade.mangafeed.data.database.models.Manga;
+import eu.kanade.mangafeed.data.source.SourceManager;
 import eu.kanade.mangafeed.data.source.base.Source;
+import eu.kanade.mangafeed.data.source.model.MangasPage;
 import rx.Observable;
 
 public class Mangahere extends Source {
 
     public static final String NAME = "Mangahere (EN)";
-    public static final String BASE_URL = "www.mangahere.co";
+    public static final String BASE_URL = "http://www.mangahere.co";
 
-    private static final String INITIAL_UPDATE_URL = "http://www.mangahere.co/directory/";
+    private static final String INITIAL_POPULAR_MANGAS_URL = "http://www.mangahere.co/directory/";
     private static final String INITIAL_SEARCH_URL = "http://www.mangahere.co/search.php?";
 
     public Mangahere(Context context) {
@@ -43,22 +44,21 @@ public class Mangahere extends Source {
         return SourceManager.MANGAHERE;
     }
 
-    @Override
-    protected String getUrlFromPageNumber(int page) {
-        return INITIAL_UPDATE_URL + page + ".htm?views.za";
-    }
-
-    @Override
-    protected String getSearchUrl(String query, int page) {
-        return INITIAL_SEARCH_URL + "name=" + query + "&page=" + page;
-    }
-
-
     @Override
     public boolean isLoginRequired() {
         return false;
     }
 
+    @Override
+    protected String getInitialPopularMangasUrl() {
+        return INITIAL_POPULAR_MANGAS_URL;
+    }
+
+    @Override
+    protected String getInitialSearchUrl(String query) {
+        return INITIAL_SEARCH_URL + "name=" + query + "&page=1";
+    }
+
     public Observable<List<String>> getGenres() {
         List<String> genres = new ArrayList<>(30);
 
@@ -98,12 +98,10 @@ public class Mangahere extends Source {
     }
 
     @Override
-    public List<Manga> parsePopularMangasFromHtml(String unparsedHtml) {
+    public List<Manga> parsePopularMangasFromHtml(Document parsedHtml) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
-
         List<Manga> mangaList = new ArrayList<>();
 
-        Elements mangaHtmlBlocks = parsedDocument.select("div.directory_list > ul > li");
+        Elements mangaHtmlBlocks = parsedHtml.select("div.directory_list > ul > li");
         for (Element currentHtmlBlock : mangaHtmlBlocks) {
             Manga currentManga = constructPopularMangaFromHtmlBlock(currentHtmlBlock);
             mangaList.add(currentManga);
@@ -127,12 +125,19 @@ public class Mangahere extends Source {
     }
 
     @Override
-    protected List<Manga> parseSearchFromHtml(String unparsedHtml) {
+    protected String parseNextPopularMangasUrl(Document parsedHtml, MangasPage page) {
-        Document parsedDocument = Jsoup.parse(unparsedHtml);
+        Element next = parsedHtml.select("div.next-page > a.next").first();
+        if (next == null)
+            return null;
 
+        return INITIAL_POPULAR_MANGAS_URL + next.attr("href");
+    }
+
+    @Override
+    protected List<Manga> parseSearchFromHtml(Document parsedHtml) {
         List<Manga> mangaList = new ArrayList<>();
 
-        Elements mangaHtmlBlocks = parsedDocument.select("div.result_search > dl");
+        Elements mangaHtmlBlocks = parsedHtml.select("div.result_search > dl");
         for (Element currentHtmlBlock : mangaHtmlBlocks) {
             Manga currentManga = constructSearchMangaFromHtmlBlock(currentHtmlBlock);
             mangaList.add(currentManga);
@@ -155,6 +160,15 @@ public class Mangahere extends Source {
         return mangaFromHtmlBlock;
     }
 
+    @Override
+    protected String parseNextSearchUrl(Document parsedHtml, MangasPage page, String query) {
+        Element next = parsedHtml.select("div.next-page > a.next").first();
+        if (next == null)
+            return null;
+
+        return BASE_URL + next.attr("href");
+    }
+
     private long parseUpdateFromElement(Element updateElement) {
         String updatedDateAsString = updateElement.text();
 

app/src/main/java/eu/kanade/mangafeed/ui/catalogue/CatalogueFragment.java

@@ -22,8 +22,8 @@ import butterknife.ButterKnife;
 import butterknife.OnItemClick;
 import eu.kanade.mangafeed.R;
 import eu.kanade.mangafeed.data.database.models.Manga;
-import eu.kanade.mangafeed.ui.manga.MangaActivity;
 import eu.kanade.mangafeed.ui.base.fragment.BaseRxFragment;
+import eu.kanade.mangafeed.ui.manga.MangaActivity;
 import eu.kanade.mangafeed.util.PageBundle;
 import eu.kanade.mangafeed.widget.EndlessScrollListener;
 import nucleus.factory.RequiresPresenter;

app/src/main/java/eu/kanade/mangafeed/ui/catalogue/CataloguePresenter.java

@@ -10,9 +10,10 @@ import java.util.concurrent.TimeUnit;
 import javax.inject.Inject;
 
 import eu.kanade.mangafeed.data.database.DatabaseHelper;
-import eu.kanade.mangafeed.data.source.SourceManager;
 import eu.kanade.mangafeed.data.database.models.Manga;
+import eu.kanade.mangafeed.data.source.SourceManager;
 import eu.kanade.mangafeed.data.source.base.Source;
+import eu.kanade.mangafeed.data.source.model.MangasPage;
 import eu.kanade.mangafeed.ui.base.presenter.BasePresenter;
 import eu.kanade.mangafeed.util.PageBundle;
 import eu.kanade.mangafeed.util.RxPager;
@@ -36,6 +37,7 @@ public class CataloguePresenter extends BasePresenter<CatalogueFragment> {
 
     private int mCurrentPage;
     private RxPager pager;
+    private MangasPage lastMangasPage;
 
     private Subscription mQueryDebouncerSubscription;
     private Subscription mMangaDetailFetchSubscription;
@@ -91,21 +93,28 @@ public class CataloguePresenter extends BasePresenter<CatalogueFragment> {
     }
 
     public void requestNext() {
-        if (getView() != null)
-            getView().showGridProgressBar();
-
         pager.requestNext(++mCurrentPage);
     }
 
     private Observable<List<Manga>> getMangaObs(int page) {
-        Observable<List<Manga>> obs;
+        MangasPage nextMangasPage = new MangasPage(page);
+        if (page != 1) {
+            if (lastMangasPage.nextPageUrl == null)
+                return Observable.empty();
+            nextMangasPage.url = lastMangasPage.nextPageUrl;
+        }
+        if (getView() != null)
+            getView().showGridProgressBar();
+
+        Observable<MangasPage> obs;
         if (mSearchMode)
-            obs = selectedSource.searchMangasFromNetwork(mSearchName, page);
+            obs = selectedSource.searchMangasFromNetwork(nextMangasPage, mSearchName);
         else
-            obs = selectedSource.pullPopularMangasFromNetwork(page);
+            obs = selectedSource.pullPopularMangasFromNetwork(nextMangasPage);
 
         return obs.subscribeOn(Schedulers.io())
-                .flatMap(Observable::from)
+                .doOnNext(mangasPage -> lastMangasPage = mangasPage)
+                .flatMap(mangasPage -> Observable.from(mangasPage.mangas))
                 .map(this::networkToLocalManga)
                 .toList();
     }