Downloading extensions from Github Repo. (#1101)

Downloading extensions from Github Repo.
This commit is contained in:
Carlos
2018-02-05 16:50:56 -05:00
committed by inorichi
parent a71c805959
commit 854112095b
46 changed files with 2319 additions and 615 deletions

View File

@@ -0,0 +1,8 @@
package eu.kanade.tachiyomi.source
import android.support.v7.preference.PreferenceScreen
interface ConfigurableSource : Source {
fun setupPreferenceScreen(screen: PreferenceScreen)
}

View File

@@ -1,30 +1,19 @@
package eu.kanade.tachiyomi.source
import android.Manifest.permission.READ_EXTERNAL_STORAGE
import android.content.Context
import android.content.pm.ApplicationInfo
import android.content.pm.PackageManager
import android.os.Environment
import dalvik.system.PathClassLoader
import eu.kanade.tachiyomi.R
import eu.kanade.tachiyomi.source.online.HttpSource
import eu.kanade.tachiyomi.source.online.YamlHttpSource
import eu.kanade.tachiyomi.source.online.english.*
import eu.kanade.tachiyomi.source.online.german.WieManga
import eu.kanade.tachiyomi.source.online.russian.Mangachan
import eu.kanade.tachiyomi.source.online.russian.Mintmanga
import eu.kanade.tachiyomi.source.online.russian.Readmanga
import eu.kanade.tachiyomi.util.hasPermission
import org.yaml.snakeyaml.Yaml
import timber.log.Timber
import java.io.File
open class SourceManager(private val context: Context) {
private val sourcesMap = mutableMapOf<Long, Source>()
init {
createSources()
createInternalSources().forEach { registerSource(it) }
}
open fun get(sourceKey: Long): Source? {
@@ -35,18 +24,16 @@ open class SourceManager(private val context: Context) {
fun getCatalogueSources() = sourcesMap.values.filterIsInstance<CatalogueSource>()
private fun createSources() {
createExtensionSources().forEach { registerSource(it) }
createYamlSources().forEach { registerSource(it) }
createInternalSources().forEach { registerSource(it) }
}
private fun registerSource(source: Source, overwrite: Boolean = false) {
internal fun registerSource(source: Source, overwrite: Boolean = false) {
if (overwrite || !sourcesMap.containsKey(source.id)) {
sourcesMap.put(source.id, source)
}
}
internal fun unregisterSource(source: Source) {
sourcesMap.remove(source.id)
}
private fun createInternalSources(): List<Source> = listOf(
LocalSource(context),
Batoto(),
@@ -60,92 +47,4 @@ open class SourceManager(private val context: Context) {
Mangasee(),
WieManga()
)
private fun createYamlSources(): List<Source> {
val sources = mutableListOf<Source>()
val parsersDir = File(Environment.getExternalStorageDirectory().absolutePath +
File.separator + context.getString(R.string.app_name), "parsers")
if (parsersDir.exists() && context.hasPermission(READ_EXTERNAL_STORAGE)) {
val yaml = Yaml()
for (file in parsersDir.listFiles().filter { it.extension == "yml" }) {
try {
val map = file.inputStream().use { yaml.loadAs(it, Map::class.java) }
sources.add(YamlHttpSource(map))
} catch (e: Exception) {
Timber.e("Error loading source from file. Bad format?", e)
}
}
}
return sources
}
private fun createExtensionSources(): List<Source> {
val pkgManager = context.packageManager
val flags = PackageManager.GET_CONFIGURATIONS or PackageManager.GET_SIGNATURES
val installedPkgs = pkgManager.getInstalledPackages(flags)
val extPkgs = installedPkgs.filter { it.reqFeatures.orEmpty().any { it.name == EXTENSION_FEATURE } }
val sources = mutableListOf<Source>()
for (pkgInfo in extPkgs) {
val appInfo = pkgManager.getApplicationInfo(pkgInfo.packageName,
PackageManager.GET_META_DATA) ?: continue
val extName = pkgManager.getApplicationLabel(appInfo).toString()
.substringAfter("Tachiyomi: ")
val version = pkgInfo.versionName
val sourceClasses = appInfo.metaData.getString(METADATA_SOURCE_CLASS)
.split(";")
.map {
val sourceClass = it.trim()
if(sourceClass.startsWith("."))
pkgInfo.packageName + sourceClass
else
sourceClass
}
val extension = Extension(extName, appInfo, version, sourceClasses)
try {
sources += loadExtension(extension)
} catch (e: Exception) {
Timber.e("Extension load error: $extName.", e)
} catch (e: LinkageError) {
Timber.e("Extension load error: $extName.", e)
}
}
return sources
}
private fun loadExtension(ext: Extension): List<Source> {
// Validate lib version
val majorLibVersion = ext.version.substringBefore('.').toInt()
if (majorLibVersion < LIB_VERSION_MIN || majorLibVersion > LIB_VERSION_MAX) {
throw Exception("Lib version is $majorLibVersion, while only versions "
+ "$LIB_VERSION_MIN to $LIB_VERSION_MAX are allowed")
}
val classLoader = PathClassLoader(ext.appInfo.sourceDir, null, context.classLoader)
return ext.sourceClasses.flatMap {
val obj = Class.forName(it, false, classLoader).newInstance()
when(obj) {
is Source -> listOf(obj)
is SourceFactory -> obj.createSources()
else -> throw Exception("Unknown source class type!")
}
}
}
class Extension(val name: String,
val appInfo: ApplicationInfo,
val version: String,
val sourceClasses: List<String>)
private companion object {
const val EXTENSION_FEATURE = "tachiyomi.extension"
const val METADATA_SOURCE_CLASS = "tachiyomi.extension.class"
const val LIB_VERSION_MIN = 1
const val LIB_VERSION_MAX = 1
}
}

View File

@@ -1,6 +1,5 @@
package eu.kanade.tachiyomi.source.online
import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.NetworkHelper
import eu.kanade.tachiyomi.network.asObservableSuccess
@@ -28,10 +27,12 @@ abstract class HttpSource : CatalogueSource {
*/
protected val network: NetworkHelper by injectLazy()
/**
* Preferences helper.
*/
protected val preferences: PreferencesHelper by injectLazy()
// /**
// * Preferences that a source may need.
// */
// val preferences: SharedPreferences by lazy {
// Injekt.get<Application>().getSharedPreferences("source_$id", Context.MODE_PRIVATE)
// }
/**
* Base url of the website without the trailing slash, like: http://mysite.com

View File

@@ -1,231 +0,0 @@
package eu.kanade.tachiyomi.source.online
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.source.model.*
import eu.kanade.tachiyomi.util.asJsoup
import eu.kanade.tachiyomi.util.attrOrText
import okhttp3.Request
import okhttp3.Response
import org.jsoup.Jsoup
import org.jsoup.nodes.Element
import java.text.SimpleDateFormat
import java.util.*
class YamlHttpSource(mappings: Map<*, *>) : HttpSource() {
val map = YamlSourceNode(mappings)
override val name: String
get() = map.name
override val baseUrl = map.host.let {
if (it.endsWith("/")) it.dropLast(1) else it
}
override val lang = map.lang.toLowerCase()
override val supportsLatest = map.latestupdates != null
override val client = when (map.client) {
"cloudflare" -> network.cloudflareClient
else -> network.client
}
override val id = map.id.let {
(it as? Int ?: (lang.toUpperCase().hashCode() + 31 * it.hashCode()) and 0x7fffffff).toLong()
}
// Ugly, but needed after the changes
var popularNextPage: String? = null
var searchNextPage: String? = null
var latestNextPage: String? = null
override fun popularMangaRequest(page: Int): Request {
val url = if (page == 1) {
popularNextPage = null
map.popular.url
} else {
popularNextPage!!
}
return when (map.popular.method?.toLowerCase()) {
"post" -> POST(url, headers, map.popular.createForm())
else -> GET(url, headers)
}
}
override fun popularMangaParse(response: Response): MangasPage {
val document = response.asJsoup()
val mangas = document.select(map.popular.manga_css).map { element ->
SManga.create().apply {
title = element.text()
setUrlWithoutDomain(element.attr("href"))
}
}
popularNextPage = map.popular.next_url_css?.let { selector ->
document.select(selector).first()?.absUrl("href")
}
return MangasPage(mangas, popularNextPage != null)
}
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val url = if (page == 1) {
searchNextPage = null
map.search.url.replace("\$query", query)
} else {
searchNextPage!!
}
return when (map.search.method?.toLowerCase()) {
"post" -> POST(url, headers, map.search.createForm())
else -> GET(url, headers)
}
}
override fun searchMangaParse(response: Response): MangasPage {
val document = response.asJsoup()
val mangas = document.select(map.search.manga_css).map { element ->
SManga.create().apply {
title = element.text()
setUrlWithoutDomain(element.attr("href"))
}
}
searchNextPage = map.search.next_url_css?.let { selector ->
document.select(selector).first()?.absUrl("href")
}
return MangasPage(mangas, searchNextPage != null)
}
override fun latestUpdatesRequest(page: Int): Request {
val url = if (page == 1) {
latestNextPage = null
map.latestupdates!!.url
} else {
latestNextPage!!
}
return when (map.latestupdates!!.method?.toLowerCase()) {
"post" -> POST(url, headers, map.latestupdates.createForm())
else -> GET(url, headers)
}
}
override fun latestUpdatesParse(response: Response): MangasPage {
val document = response.asJsoup()
val mangas = document.select(map.latestupdates!!.manga_css).map { element ->
SManga.create().apply {
title = element.text()
setUrlWithoutDomain(element.attr("href"))
}
}
popularNextPage = map.latestupdates.next_url_css?.let { selector ->
document.select(selector).first()?.absUrl("href")
}
return MangasPage(mangas, popularNextPage != null)
}
override fun mangaDetailsParse(response: Response): SManga {
val document = response.asJsoup()
val manga = SManga.create()
with(map.manga) {
val pool = parts.get(document)
manga.author = author?.process(document, pool)
manga.artist = artist?.process(document, pool)
manga.description = summary?.process(document, pool)
manga.thumbnail_url = cover?.process(document, pool)
manga.genre = genres?.process(document, pool)
manga.status = status?.getStatus(document, pool) ?: SManga.UNKNOWN
}
return manga
}
override fun chapterListParse(response: Response): List<SChapter> {
val document = response.asJsoup()
val chapters = mutableListOf<SChapter>()
with(map.chapters) {
val pool = emptyMap<String, Element>()
val dateFormat = SimpleDateFormat(date?.format, Locale.ENGLISH)
for (element in document.select(chapter_css)) {
val chapter = SChapter.create()
element.select(title).first().let {
chapter.name = it.text()
chapter.setUrlWithoutDomain(it.attr("href"))
}
val dateElement = element.select(date?.select).first()
chapter.date_upload = date?.getDate(dateElement, pool, dateFormat)?.time ?: 0
chapters.add(chapter)
}
}
return chapters
}
override fun pageListParse(response: Response): List<Page> {
val body = response.body()!!.string()
val url = response.request().url().toString()
val pages = mutableListOf<Page>()
val document by lazy { Jsoup.parse(body, url) }
with(map.pages) {
// Capture a list of values where page urls will be resolved.
val capturedPages = if (pages_regex != null)
pages_regex!!.toRegex().findAll(body).map { it.value }.toList()
else if (pages_css != null)
document.select(pages_css).map { it.attrOrText(pages_attr!!) }
else
null
// For each captured value, obtain the url and create a new page.
capturedPages?.forEach { value ->
// If the captured value isn't an url, we have to use replaces with the chapter url.
val pageUrl = if (replace != null && replacement != null)
url.replace(replace!!.toRegex(), replacement!!.replace("\$value", value))
else
value
pages.add(Page(pages.size, pageUrl))
}
// Capture a list of images.
val capturedImages = if (image_regex != null)
image_regex!!.toRegex().findAll(body).map { it.groups[1]?.value }.toList()
else if (image_css != null)
document.select(image_css).map { it.absUrl(image_attr) }
else
null
// Assign the image url to each page
capturedImages?.forEachIndexed { i, url ->
val page = pages.getOrElse(i) { Page(i, "").apply { pages.add(this) } }
page.imageUrl = url
}
}
return pages
}
override fun imageUrlParse(response: Response): String {
val body = response.body()!!.string()
val url = response.request().url().toString()
with(map.pages) {
return if (image_regex != null)
image_regex!!.toRegex().find(body)!!.groups[1]!!.value
else if (image_css != null)
Jsoup.parse(body, url).select(image_css).first().absUrl(image_attr)
else
throw Exception("image_regex and image_css are null")
}
}
}

View File

@@ -1,234 +0,0 @@
@file:Suppress("UNCHECKED_CAST")
package eu.kanade.tachiyomi.source.online
import eu.kanade.tachiyomi.source.model.SManga
import okhttp3.FormBody
import okhttp3.RequestBody
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import java.text.ParseException
import java.text.SimpleDateFormat
import java.util.*
private fun toMap(map: Any?) = map as? Map<String, Any?>
class YamlSourceNode(uncheckedMap: Map<*, *>) {
val map = toMap(uncheckedMap)!!
val id: Any by map
val name: String by map
val host: String by map
val lang: String by map
val client: String?
get() = map["client"] as? String
val popular = PopularNode(toMap(map["popular"])!!)
val latestupdates = toMap(map["latest_updates"])?.let { LatestUpdatesNode(it) }
val search = SearchNode(toMap(map["search"])!!)
val manga = MangaNode(toMap(map["manga"])!!)
val chapters = ChaptersNode(toMap(map["chapters"])!!)
val pages = PagesNode(toMap(map["pages"])!!)
}
interface RequestableNode {
val map: Map<String, Any?>
val url: String
get() = map["url"] as String
val method: String?
get() = map["method"] as? String
val payload: Map<String, String>?
get() = map["payload"] as? Map<String, String>
fun createForm(): RequestBody {
return FormBody.Builder().apply {
payload?.let {
for ((key, value) in it) {
add(key, value)
}
}
}.build()
}
}
class PopularNode(override val map: Map<String, Any?>): RequestableNode {
val manga_css: String by map
val next_url_css: String?
get() = map["next_url_css"] as? String
}
class LatestUpdatesNode(override val map: Map<String, Any?>): RequestableNode {
val manga_css: String by map
val next_url_css: String?
get() = map["next_url_css"] as? String
}
class SearchNode(override val map: Map<String, Any?>): RequestableNode {
val manga_css: String by map
val next_url_css: String?
get() = map["next_url_css"] as? String
}
class MangaNode(private val map: Map<String, Any?>) {
val parts = CacheNode(toMap(map["parts"]) ?: emptyMap())
val artist = toMap(map["artist"])?.let { SelectableNode(it) }
val author = toMap(map["author"])?.let { SelectableNode(it) }
val summary = toMap(map["summary"])?.let { SelectableNode(it) }
val status = toMap(map["status"])?.let { StatusNode(it) }
val genres = toMap(map["genres"])?.let { SelectableNode(it) }
val cover = toMap(map["cover"])?.let { CoverNode(it) }
}
class ChaptersNode(private val map: Map<String, Any?>) {
val chapter_css: String by map
val title: String by map
val date = toMap(toMap(map["date"]))?.let { DateNode(it) }
}
class CacheNode(private val map: Map<String, Any?>) {
fun get(document: Document) = map.mapValues { document.select(it.value as String).first() }
}
open class SelectableNode(private val map: Map<String, Any?>) {
val select: String by map
val from: String?
get() = map["from"] as? String
open val attr: String?
get() = map["attr"] as? String
val capture: String?
get() = map["capture"] as? String
fun process(document: Element, cache: Map<String, Element>): String {
val parent = from?.let { cache[it] } ?: document
val node = parent.select(select).first()
var text = attr?.let { node.attr(it) } ?: node.text()
capture?.let {
text = Regex(it).find(text)?.groupValues?.get(1) ?: text
}
return text
}
}
class StatusNode(private val map: Map<String, Any?>) : SelectableNode(map) {
val complete: String?
get() = map["complete"] as? String
val ongoing: String?
get() = map["ongoing"] as? String
val licensed: String?
get() = map["licensed"] as? String
fun getStatus(document: Element, cache: Map<String, Element>): Int {
val text = process(document, cache)
complete?.let {
if (text.contains(it)) return SManga.COMPLETED
}
ongoing?.let {
if (text.contains(it)) return SManga.ONGOING
}
licensed?.let {
if (text.contains(it)) return SManga.LICENSED
}
return SManga.UNKNOWN
}
}
class CoverNode(private val map: Map<String, Any?>) : SelectableNode(map) {
override val attr: String?
get() = map["attr"] as? String ?: "src"
}
class DateNode(private val map: Map<String, Any?>) : SelectableNode(map) {
val format: String by map
fun getDate(document: Element, cache: Map<String, Element>, formatter: SimpleDateFormat): Date {
val text = process(document, cache)
try {
return formatter.parse(text)
} catch (exception: ParseException) {}
for (i in 0..7) {
(map["day$i"] as? List<String>)?.let {
it.find { it.toRegex().containsMatchIn(text) }?.let {
return Calendar.getInstance().apply { add(Calendar.DATE, -i) }.time
}
}
}
return Date(0)
}
}
class PagesNode(private val map: Map<String, Any?>) {
val pages_regex: String?
get() = map["pages_regex"] as? String
val pages_css: String?
get() = map["pages_css"] as? String
val pages_attr: String?
get() = map["pages_attr"] as? String ?: "value"
val replace: String?
get() = map["url_replace"] as? String
val replacement: String?
get() = map["url_replacement"] as? String
val image_regex: String?
get() = map["image_regex"] as? String
val image_css: String?
get() = map["image_css"] as? String
val image_attr: String
get() = map["image_attr"] as? String ?: "src"
}

View File

@@ -1,6 +1,7 @@
package eu.kanade.tachiyomi.source.online.english
import android.text.Html
import eu.kanade.tachiyomi.data.preference.PreferencesHelper
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.network.asObservable
@@ -9,14 +10,11 @@ import eu.kanade.tachiyomi.source.online.LoginSource
import eu.kanade.tachiyomi.source.online.ParsedHttpSource
import eu.kanade.tachiyomi.util.asJsoup
import eu.kanade.tachiyomi.util.selectText
import okhttp3.FormBody
import okhttp3.HttpUrl
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import okhttp3.*
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import rx.Observable
import uy.kohesive.injekt.injectLazy
import java.net.URI
import java.text.ParseException
import java.text.SimpleDateFormat
@@ -25,6 +23,9 @@ import java.util.regex.Pattern
class Batoto : ParsedHttpSource(), LoginSource {
// TODO remove
private val preferences: PreferencesHelper by injectLazy()
override val id: Long = 1
override val name = "Batoto"