From 36265472f83d07dd24b621aadb2d8ede4579ac04 Mon Sep 17 00:00:00 2001 From: inorichi Date: Fri, 9 Oct 2015 22:08:36 +0200 Subject: [PATCH] Move generic methods to Source --- .../eu/kanade/mangafeed/sources/Batoto.java | 282 ++---------------- .../eu/kanade/mangafeed/sources/Source.java | 102 ++++++- 2 files changed, 134 insertions(+), 250 deletions(-) diff --git a/app/src/main/java/eu/kanade/mangafeed/sources/Batoto.java b/app/src/main/java/eu/kanade/mangafeed/sources/Batoto.java index 42c18ce75..6944bea20 100644 --- a/app/src/main/java/eu/kanade/mangafeed/sources/Batoto.java +++ b/app/src/main/java/eu/kanade/mangafeed/sources/Batoto.java @@ -13,7 +13,6 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Locale; -import java.util.concurrent.atomic.AtomicInteger; import eu.kanade.mangafeed.data.caches.CacheManager; import eu.kanade.mangafeed.data.helpers.NetworkHelper; @@ -21,45 +20,27 @@ import eu.kanade.mangafeed.data.helpers.SourceManager; import eu.kanade.mangafeed.data.models.Chapter; import eu.kanade.mangafeed.data.models.Manga; import rx.Observable; -import rx.schedulers.Schedulers; public class Batoto extends Source { public static final String NAME = "Batoto (EN)"; public static final String BASE_URL = "www.bato.to"; - public static final String INITIAL_UPDATE_URL = "http://bato.to/search_ajax?order_cond=views&order=desc&p=1"; - - private static final Headers REQUEST_HEADERS = constructRequestHeaders(); - private static Headers constructRequestHeaders() { - Headers.Builder headerBuilder = new Headers.Builder(); - headerBuilder.add("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64)"); - headerBuilder.add("Cookie", "lang_option=English"); - - return headerBuilder.build(); - } - - private NetworkHelper mNetworkService; - private CacheManager mCacheManager; + public static final String INITIAL_UPDATE_URL = + "http://bato.to/search_ajax?order_cond=views&order=desc&p="; public Batoto(NetworkHelper networkService, CacheManager cacheManager) { - mNetworkService = networkService; - mCacheManager = cacheManager; + super(networkService, cacheManager); } - public Observable getName() { - return Observable.just(NAME); - } - - public Observable getBaseUrl() { - return Observable.just(BASE_URL); - } - - public Observable getInitialUpdateUrl() { - return Observable.just(INITIAL_UPDATE_URL); + @Override + protected Headers.Builder headersBuilder() { + Headers.Builder builder = super.headersBuilder(); + builder.add("Cookie", "lang_option=English"); + return builder; } public Observable> getGenres() { - List genres = new ArrayList(38); + List genres = new ArrayList<>(38); genres.add("4-Koma"); genres.add("Action"); @@ -103,21 +84,24 @@ public class Batoto extends Source { return Observable.just(genres); } - public String getUrlFromPageNumber(int page) { - if (page == 1) - return INITIAL_UPDATE_URL; - - return INITIAL_UPDATE_URL.substring(0, INITIAL_UPDATE_URL.length() - 1) + page; + @Override + protected int getSource() { + return SourceManager.BATOTO; } - public Observable> pullPopularMangasFromNetwork(int page) { - String url = getUrlFromPageNumber(page); - return mNetworkService - .getStringResponse(url, mNetworkService.NULL_CACHE_CONTROL, REQUEST_HEADERS) - .flatMap(response -> Observable.just(parsePopularMangasFromHtml(response))); + @Override + protected String getUrlFromPageNumber(int page) { + return INITIAL_UPDATE_URL + page; } - private List parsePopularMangasFromHtml(String unparsedHtml) { + @Override + protected String getMangaUrl(String defaultMangaUrl) { + String mangaId = defaultMangaUrl.substring(defaultMangaUrl.lastIndexOf("r") + 1); + return "http://bato.to/comic_pop?id=" + mangaId; + } + + @Override + public List parsePopularMangasFromHtml(String unparsedHtml) { Document parsedDocument = Jsoup.parse(unparsedHtml); List updatedMangaList = new ArrayList<>(); @@ -139,7 +123,7 @@ public class Batoto extends Source { Element nameElement = urlElement; Element updateElement = htmlBlock.select("td").get(5); - mangaFromHtmlBlock.source = SourceManager.BATOTO; + mangaFromHtmlBlock.source = getSource(); if (urlElement != null) { String fieldUrl = urlElement.attr("href"); @@ -171,15 +155,8 @@ public class Batoto extends Source { return 0; } - public Observable pullMangaFromNetwork(final String mangaUrl) { - String mangaId = mangaUrl.substring(mangaUrl.lastIndexOf("r") + 1); - - return mNetworkService - .getStringResponse("http://bato.to/comic_pop?id=" + mangaId, mNetworkService.NULL_CACHE_CONTROL, REQUEST_HEADERS) - .flatMap(unparsedHtml -> Observable.just(parseHtmlToManga(mangaUrl, unparsedHtml))); - } - - private Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { + @Override + protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml) { Document parsedDocument = Jsoup.parse(unparsedHtml); Elements artistElements = parsedDocument.select("a[href^=http://bato.to/search?artist_name]"); @@ -229,14 +206,8 @@ public class Batoto extends Source { return newManga; } - public Observable> pullChaptersFromNetwork(String mangaUrl) { - return mNetworkService - .getStringResponse(mangaUrl, mNetworkService.NULL_CACHE_CONTROL, REQUEST_HEADERS) - .flatMap(unparsedHtml -> - Observable.just(parseHtmlToChapters(unparsedHtml))); - } - - private List parseHtmlToChapters(String unparsedHtml) { + @Override + protected List parseHtmlToChapters(String unparsedHtml) { Document parsedDocument = Jsoup.parse(unparsedHtml); List chapterList = new ArrayList<>(); @@ -291,42 +262,11 @@ public class Batoto extends Source { return 0; } - public Observable pullImageUrlsFromNetwork(final String chapterUrl) { - final List temporaryCachedImageUrls = new ArrayList<>(); - - return mCacheManager.getImageUrlsFromDiskCache(chapterUrl) - .onErrorResumeNext(throwable -> { - return getImageUrlsFromNetwork(chapterUrl) - .doOnNext(imageUrl -> temporaryCachedImageUrls.add(imageUrl)) - .doOnCompleted(mCacheManager.putImageUrlsToDiskCache(chapterUrl, temporaryCachedImageUrls)); - }) - .onBackpressureBuffer(); - } - - public Observable getImageUrlsFromNetwork(final String chapterUrl) { - return mNetworkService - .getStringResponse(chapterUrl, mNetworkService.NULL_CACHE_CONTROL, REQUEST_HEADERS) - .flatMap(unparsedHtml -> Observable.from(parseHtmlToPageUrls(unparsedHtml))) - .buffer(3) - .concatMap(batchedPageUrls -> { - List> imageUrlObservables = new ArrayList<>(); - for (String pageUrl : batchedPageUrls) { - Observable temporaryObservable = mNetworkService - .getStringResponse(pageUrl, mNetworkService.NULL_CACHE_CONTROL, REQUEST_HEADERS) - .flatMap(unparsedHtml -> Observable.just(parseHtmlToImageUrl(unparsedHtml))) - .subscribeOn(Schedulers.io()); - - imageUrlObservables.add(temporaryObservable); - } - - return Observable.merge(imageUrlObservables); - }); - } - - private List parseHtmlToPageUrls(String unparsedHtml) { + @Override + protected List parseHtmlToPageUrls(String unparsedHtml) { Document parsedDocument = Jsoup.parse(unparsedHtml); - List pageUrlList = new ArrayList(); + List pageUrlList = new ArrayList<>(); Elements pageUrlElements = parsedDocument.getElementById("page_select").getElementsByTag("option"); for (Element pageUrlElement : pageUrlElements) { @@ -336,7 +276,8 @@ public class Batoto extends Source { return pageUrlList; } - private String parseHtmlToImageUrl(String unparsedHtml) { + @Override + protected String parseHtmlToImageUrl(String unparsedHtml) { int beginIndex = unparsedHtml.indexOf("", beginIndex); String trimmedHtml = unparsedHtml.substring(beginIndex, endIndex); @@ -348,162 +289,5 @@ public class Batoto extends Source { return imageElement.attr("src"); } - private static String INITIAL_DATABASE_URL_1 = "http://bato.to/comic_pop?id=1"; - private static String INITIAL_DATABASE_URL_2 = "http://bato.to/search_ajax?order_cond=views&order=desc&p=1"; - - private static AtomicInteger mCounter = new AtomicInteger(1); - - /* - public Observable recursivelyConstructDatabase(final String url) { - return mNetworkService - .getResponse(url, NetworkUtil.NULL_CACHE_CONTROL, REQUEST_HEADERS) - .flatMap(new Func1>() { - @Override - public Observable call(Response response) { - return mNetworkService.mapResponseToString(response); - } - }) - .flatMap(new Func1>() { - @Override - public Observable call(String unparsedHtml) { - return Observable.just(parseEnglish_Batoto(unparsedHtml)); - } - }); - } - - - private String parseEnglish_Batoto(String unparsedHtml) { - if (!unparsedHtml.equals("wtf?")) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - - Manga newManga = new Manga(); - - Element temporaryElementOne = parsedDocument.getElementsByTag("a").first(); - Element temporaryElementTwo = parsedDocument.select("a[href^=http://bato.to/forums/forum/]").first(); - Element temporaryElementThree = parsedDocument.select("img[src^=http://img.batoto.net/forums/uploads/]").first(); - Elements temporaryElementsFour = parsedDocument.select("img[src=http://bato.to/forums/public/style_images/master/bullet_black.png]"); - - String fieldSource = English_Batoto.NAME; - newManga.setSource(fieldSource); - - String fieldUrl = "http://bato.to" + temporaryElementOne.attr("href"); - newManga.setUrl(fieldUrl); - - String fieldName = temporaryElementTwo.text(); - int startIndex = "Go to ".length(); - int endIndex = fieldName.lastIndexOf(" Forums!"); - newManga.setName(fieldName.substring(startIndex, endIndex)); - - String fieldThumbnailUrl = temporaryElementThree.attr("src"); - newManga.setThumbnailUrl(fieldThumbnailUrl); - - String fieldGenres = ""; - for (int index = 0; index < temporaryElementsFour.size(); index++) { - String currentGenre = temporaryElementsFour.get(index).attr("alt"); - - if (index < temporaryElementsFour.size() - 1) { - fieldGenres += currentGenre + ", "; - } else { - fieldGenres += currentGenre; - } - } - newManga.setGenre(fieldGenres); - - boolean fieldIsCompleted = unparsedHtml.contains("Complete"); - newManga.setCompleted(fieldIsCompleted); - - mQueryManager.createManga(newManga) - .toBlocking() - .single(); - } - - return "http://bato.to/comic_pop?id=" + mCounter.incrementAndGet(); - } - - private String parseEnglish_Batoto_Views(String unparsedHtml) { - if (!unparsedHtml.contains("No (more) comics found!")) { - Document parsedDocument = Jsoup.parse(unparsedHtml); - - List> updateList = new ArrayList>(); - Elements mangaElements = parsedDocument.select("tr:not([id]):not([class])"); - for (Element mangaElement : mangaElements) { - Element temporaryElementOne = mangaElement.select("a[href^=http://bato.to]").first(); - Element temporaryElementTwo = mangaElement.select("td").get(3); - String temporaryString = temporaryElementTwo.text(); - - String fieldUrl = temporaryElementOne.attr("href"); - - String fieldView = null; - if (temporaryString.contains("m")) { - temporaryString = temporaryString.replace("m", ""); - - int viewsAsNumber = (int)(Double.valueOf(temporaryString) * 1000000); - fieldView = String.valueOf(viewsAsNumber); - } else if (temporaryString.contains("k")) { - temporaryString = temporaryString.replace("k", ""); - - int viewsAsNumber = (int)(Double.valueOf(temporaryString) * 1000); - fieldView = String.valueOf(viewsAsNumber); - } else { - int viewsAsNumber = (int)(Double.valueOf(temporaryString) * 1); - fieldView = String.valueOf(viewsAsNumber); - } - - ContentValues fieldRanking = new ContentValues(1); - fieldRanking.put(LibraryContract.Manga.COLUMN_RANK, fieldView); - - updateList.add(Pair.create(fieldUrl, fieldRanking)); - } - - mQueryManager.beginLibraryTransaction(); - try { - for (Pair currentUpdate : updateList) { - mQueryManager.updateManga(currentUpdate.second, LibraryContract.Manga.COLUMN_URL + " = ?", new String[]{currentUpdate.first}) - .toBlocking() - .single(); - } - - mQueryManager.setLibraryTransactionSuccessful(); - } finally { - mQueryManager.endLibraryTransaction(); - } - - return "http://bato.to/search_ajax?order_cond=views&order=desc&p=" + mCounter.incrementAndGet(); - } - - return null; - } - - public void reorderEnglish_Batoto_Rankings() { - List mangaList = mQueryManager.retrieveAllMangaAsStream( - null, - LibraryContract.Manga.COLUMN_SOURCE + " = ?", - new String[]{NAME}, - null, - null, - LibraryContract.Manga.COLUMN_RANK + " DESC", - null - ) - .toList() - .toBlocking() - .single(); - - for (int index = 0; index < mangaList.size(); index++) { - mangaList.get(index).setRank(index + 1); - } - - mQueryManager.beginLibraryTransaction(); - try { - for (Manga currentManga : mangaList) { - mQueryManager.createManga(currentManga) - .toBlocking() - .single(); - } - mQueryManager.setLibraryTransactionSuccessful(); - } finally { - mQueryManager.endLibraryTransaction(); - } - } - */ } diff --git a/app/src/main/java/eu/kanade/mangafeed/sources/Source.java b/app/src/main/java/eu/kanade/mangafeed/sources/Source.java index 4d0d231aa..4ff121352 100644 --- a/app/src/main/java/eu/kanade/mangafeed/sources/Source.java +++ b/app/src/main/java/eu/kanade/mangafeed/sources/Source.java @@ -1,5 +1,105 @@ package eu.kanade.mangafeed.sources; -public class Source { +import com.squareup.okhttp.Headers; + +import java.util.ArrayList; +import java.util.List; + +import eu.kanade.mangafeed.data.caches.CacheManager; +import eu.kanade.mangafeed.data.helpers.NetworkHelper; +import eu.kanade.mangafeed.data.models.Chapter; +import eu.kanade.mangafeed.data.models.Manga; +import rx.Observable; +import rx.schedulers.Schedulers; + +public abstract class Source { + + protected NetworkHelper mNetworkService; + protected CacheManager mCacheManager; + protected Headers mRequestHeaders; + + public Source(NetworkHelper networkService, CacheManager cacheManager) { + mNetworkService = networkService; + mCacheManager = cacheManager; + mRequestHeaders = headersBuilder().build(); + } + + // Default headers, it can be overriden by children or add new keys + protected Headers.Builder headersBuilder() { + Headers.Builder builder = new Headers.Builder(); + builder.add("User-Agent", "Mozilla/5.0 (Windows NT 6.3; WOW64)"); + return builder; + } + + // Get the most popular mangas from the source + public Observable> pullPopularMangasFromNetwork(int page) { + String url = getUrlFromPageNumber(page); + return mNetworkService + .getStringResponse(url, mNetworkService.NULL_CACHE_CONTROL, mRequestHeaders) + .flatMap(response -> Observable.just(parsePopularMangasFromHtml(response))); + } + + // Get manga details from the source + public Observable pullMangaFromNetwork(final String mangaUrl) { + return mNetworkService + .getStringResponse(getMangaUrl(mangaUrl), mNetworkService.NULL_CACHE_CONTROL, mRequestHeaders) + .flatMap(unparsedHtml -> Observable.just(parseHtmlToManga(mangaUrl, unparsedHtml))); + } + + // Get chapter list of a manga from the source + public Observable> pullChaptersFromNetwork(String mangaUrl) { + return mNetworkService + .getStringResponse(mangaUrl, mNetworkService.NULL_CACHE_CONTROL, mRequestHeaders) + .flatMap(unparsedHtml -> + Observable.just(parseHtmlToChapters(unparsedHtml))); + } + + // Get the URLs of the images of a chapter + public Observable getImageUrlsFromNetwork(final String chapterUrl) { + return mNetworkService + .getStringResponse(chapterUrl, mNetworkService.NULL_CACHE_CONTROL, mRequestHeaders) + .flatMap(unparsedHtml -> Observable.from(parseHtmlToPageUrls(unparsedHtml))) + .buffer(3) + .concatMap(batchedPageUrls -> { + List> imageUrlObservables = new ArrayList<>(); + for (String pageUrl : batchedPageUrls) { + Observable temporaryObservable = mNetworkService + .getStringResponse(pageUrl, mNetworkService.NULL_CACHE_CONTROL, mRequestHeaders) + .flatMap(unparsedHtml -> Observable.just(parseHtmlToImageUrl(unparsedHtml))) + .subscribeOn(Schedulers.io()); + + imageUrlObservables.add(temporaryObservable); + } + + return Observable.merge(imageUrlObservables); + }); + } + + // Store the URLs of a chapter in the cache + public Observable pullImageUrlsFromNetwork(final String chapterUrl) { + final List temporaryCachedImageUrls = new ArrayList<>(); + + return mCacheManager.getImageUrlsFromDiskCache(chapterUrl) + .onErrorResumeNext(throwable -> { + return getImageUrlsFromNetwork(chapterUrl) + .doOnNext(imageUrl -> temporaryCachedImageUrls.add(imageUrl)) + .doOnCompleted(mCacheManager.putImageUrlsToDiskCache(chapterUrl, temporaryCachedImageUrls)); + }) + .onBackpressureBuffer(); + } + + // Get the URL to the details of a manga, useful if the source provides some kind of API or fast calls + protected String getMangaUrl(String defaultMangaUrl) { + return defaultMangaUrl; + } + + abstract protected int getSource(); + abstract protected String getUrlFromPageNumber(int page); + abstract protected List parsePopularMangasFromHtml(String unparsedHtml); + abstract protected Manga parseHtmlToManga(String mangaUrl, String unparsedHtml); + abstract protected List parseHtmlToChapters(String unparsedHtml); + abstract protected List parseHtmlToPageUrls(String unparsedHtml); + abstract protected String parseHtmlToImageUrl(String unparsedHtml); + }