mirror of
https://github.com/aniyomiorg/aniyomi.git
synced 2024-11-24 21:58:34 +03:00
Test solving Cloudflare's challenge with WebView
This commit is contained in:
inorichi
parent
55bf1c31a6
commit
f1f6a2b341
8 changed files with 250 additions and 194 deletions
|
|
@ -7,9 +7,9 @@ import eu.kanade.tachiyomi.data.database.models.Track
|
|||
import eu.kanade.tachiyomi.data.preference.getOrDefault
|
||||
import eu.kanade.tachiyomi.data.track.TrackService
|
||||
import eu.kanade.tachiyomi.data.track.model.TrackSearch
|
||||
import okhttp3.HttpUrl
|
||||
import rx.Completable
|
||||
import rx.Observable
|
||||
import java.net.URI
|
||||
|
||||
class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
|
||||
|
||||
|
|
@ -114,23 +114,23 @@ class Myanimelist(private val context: Context, id: Int) : TrackService(id) {
|
|||
override fun logout() {
|
||||
super.logout()
|
||||
preferences.trackToken(this).delete()
|
||||
networkService.cookies.remove(URI(BASE_URL))
|
||||
networkService.cookieManager.remove(HttpUrl.parse(BASE_URL)!!)
|
||||
}
|
||||
|
||||
override val isLogged: Boolean
|
||||
get() = !getUsername().isEmpty() &&
|
||||
!getPassword().isEmpty() &&
|
||||
checkCookies(URI(BASE_URL)) &&
|
||||
checkCookies() &&
|
||||
!getCSRF().isEmpty()
|
||||
|
||||
private fun getCSRF(): String = preferences.trackToken(this).getOrDefault()
|
||||
|
||||
private fun saveCSRF(csrf: String) = preferences.trackToken(this).set(csrf)
|
||||
|
||||
private fun checkCookies(uri: URI): Boolean {
|
||||
private fun checkCookies(): Boolean {
|
||||
var ckCount = 0
|
||||
|
||||
for (ck in networkService.cookies.get(uri)) {
|
||||
val url = HttpUrl.parse(BASE_URL)!!
|
||||
for (ck in networkService.cookieManager.get(url)) {
|
||||
if (ck.name() == USER_SESSION_COOKIE || ck.name() == LOGGED_IN_COOKIE)
|
||||
ckCount++
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,63 @@
|
|||
package eu.kanade.tachiyomi.network
|
||||
|
||||
import android.content.Context
|
||||
import android.os.Build
|
||||
import android.webkit.CookieManager
|
||||
import android.webkit.CookieSyncManager
|
||||
import okhttp3.Cookie
|
||||
import okhttp3.CookieJar
|
||||
import okhttp3.HttpUrl
|
||||
|
||||
class AndroidCookieJar(context: Context) : CookieJar {
|
||||
|
||||
private val manager = CookieManager.getInstance()
|
||||
|
||||
private val syncManager by lazy { CookieSyncManager.createInstance(context) }
|
||||
|
||||
override fun saveFromResponse(url: HttpUrl, cookies: MutableList<Cookie>) {
|
||||
val urlString = url.toString()
|
||||
|
||||
for (cookie in cookies) {
|
||||
manager.setCookie(urlString, cookie.toString())
|
||||
}
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
|
||||
syncManager.sync()
|
||||
}
|
||||
}
|
||||
|
||||
override fun loadForRequest(url: HttpUrl): List<Cookie> {
|
||||
return get(url)
|
||||
}
|
||||
|
||||
fun get(url: HttpUrl): List<Cookie> {
|
||||
val cookies = manager.getCookie(url.toString())
|
||||
|
||||
return if (cookies != null && !cookies.isEmpty()) {
|
||||
cookies.split(";").mapNotNull { Cookie.parse(url, it) }
|
||||
} else {
|
||||
emptyList()
|
||||
}
|
||||
}
|
||||
|
||||
fun remove(url: HttpUrl) {
|
||||
val cookies = manager.getCookie(url.toString()) ?: return
|
||||
val domain = ".${url.host()}"
|
||||
cookies.split(";")
|
||||
.map { it.substringBefore("=") }
|
||||
.onEach { manager.setCookie(domain, "$it=;Max-Age=-1") }
|
||||
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.LOLLIPOP) {
|
||||
syncManager.sync()
|
||||
}
|
||||
}
|
||||
|
||||
fun removeAll() {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
|
||||
manager.removeAllCookies {}
|
||||
} else {
|
||||
manager.removeAllCookie()
|
||||
syncManager.sync()
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,31 +1,32 @@
|
|||
package eu.kanade.tachiyomi.network
|
||||
|
||||
import com.squareup.duktape.Duktape
|
||||
import okhttp3.*
|
||||
import android.annotation.SuppressLint
|
||||
import android.content.Context
|
||||
import android.os.Handler
|
||||
import android.os.HandlerThread
|
||||
import android.webkit.WebResourceResponse
|
||||
import android.webkit.WebView
|
||||
import eu.kanade.tachiyomi.util.WebViewClientCompat
|
||||
import okhttp3.Interceptor
|
||||
import okhttp3.Request
|
||||
import okhttp3.Response
|
||||
import timber.log.Timber
|
||||
import java.io.IOException
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
class CloudflareInterceptor : Interceptor {
|
||||
|
||||
private val operationPattern = Regex("""setTimeout\(function\(\)\{\s+(var (?:\w,)+f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n""")
|
||||
|
||||
private val passPattern = Regex("""name="pass" value="(.+?)"""")
|
||||
|
||||
private val challengePattern = Regex("""name="jschl_vc" value="(\w+)"""")
|
||||
|
||||
private val sPattern = Regex("""name="s" value="([^"]+)""")
|
||||
|
||||
private val kPattern = Regex("""k\s+=\s+'([^']+)';""")
|
||||
class CloudflareInterceptor(private val context: Context) : Interceptor {
|
||||
|
||||
private val serverCheck = arrayOf("cloudflare-nginx", "cloudflare")
|
||||
|
||||
private interface IBase64 {
|
||||
fun decode(input: String): String
|
||||
private val handler by lazy {
|
||||
val thread = HandlerThread("WebViewThread").apply {
|
||||
uncaughtExceptionHandler = Thread.UncaughtExceptionHandler { _, e ->
|
||||
Timber.e(e)
|
||||
}
|
||||
|
||||
private val b64: IBase64 = object : IBase64 {
|
||||
override fun decode(input: String): String {
|
||||
return okio.ByteString.decodeBase64(input)!!.utf8()
|
||||
start()
|
||||
}
|
||||
Handler(thread.looper)
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
|
|
@ -34,8 +35,14 @@ class CloudflareInterceptor : Interceptor {
|
|||
|
||||
// Check if Cloudflare anti-bot is on
|
||||
if (response.code() == 503 && response.header("Server") in serverCheck) {
|
||||
return try {
|
||||
chain.proceed(resolveChallenge(response))
|
||||
try {
|
||||
response.close()
|
||||
if (resolveWithWebView(chain.request())) {
|
||||
// Retry original request
|
||||
return chain.proceed(chain.request())
|
||||
} else {
|
||||
throw Exception("Failed resolving Cloudflare challenge")
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
// Because OkHttp's enqueue only handles IOExceptions, wrap the exception so that
|
||||
// we don't crash the entire app
|
||||
|
|
@ -46,65 +53,76 @@ class CloudflareInterceptor : Interceptor {
|
|||
return response
|
||||
}
|
||||
|
||||
private fun resolveChallenge(response: Response): Request {
|
||||
Duktape.create().use { duktape ->
|
||||
val originalRequest = response.request()
|
||||
val url = originalRequest.url()
|
||||
val domain = url.host()
|
||||
val content = response.body()!!.string()
|
||||
|
||||
// CloudFlare requires waiting 4 seconds before resolving the challenge
|
||||
Thread.sleep(4000)
|
||||
|
||||
val operation = operationPattern.find(content)?.groups?.get(1)?.value
|
||||
val challenge = challengePattern.find(content)?.groups?.get(1)?.value
|
||||
val pass = passPattern.find(content)?.groups?.get(1)?.value
|
||||
val s = sPattern.find(content)?.groups?.get(1)?.value
|
||||
|
||||
// If `k` is null, it uses old methods.
|
||||
val k = kPattern.find(content)?.groups?.get(1)?.value ?: ""
|
||||
val innerHTMLValue = Regex("""<div(.*)id="$k"(.*)>(.*)</div>""")
|
||||
.find(content)?.groups?.get(3)?.value ?: ""
|
||||
|
||||
if (operation == null || challenge == null || pass == null || s == null) {
|
||||
throw Exception("Failed resolving Cloudflare challenge")
|
||||
private fun isChallengeResolverUrl(url: String): Boolean {
|
||||
return "chk_jschl" in url
|
||||
}
|
||||
|
||||
// Export native Base64 decode function to js object.
|
||||
duktape.set("b64", IBase64::class.java, b64)
|
||||
@SuppressLint("SetJavaScriptEnabled")
|
||||
private fun resolveWithWebView(request: Request): Boolean {
|
||||
val latch = CountDownLatch(1)
|
||||
|
||||
// Return simulated innerHTML when call DOM.
|
||||
val simulatedDocumentJS = """var document = { getElementById: function (x) { return { innerHTML: "$innerHTMLValue" }; } }"""
|
||||
var result = false
|
||||
var isResolvingChallenge = false
|
||||
|
||||
val js = operation
|
||||
.replace(Regex("""a\.value = (.+\.toFixed\(10\);).+"""), "$1")
|
||||
.replace(Regex("""\s{3,}[a-z](?: = |\.).+"""), "")
|
||||
.replace("t.length", "${domain.length}")
|
||||
.replace("\n", "")
|
||||
val requestUrl = request.url().toString()
|
||||
val headers = request.headers().toMultimap().mapValues { it.value.getOrNull(0) ?: "" }
|
||||
|
||||
val result = duktape.evaluate("""$simulatedDocumentJS;$ATOB_JS;var t="$domain";$js""") as String
|
||||
handler.post {
|
||||
val view = WebView(context)
|
||||
view.settings.javaScriptEnabled = true
|
||||
view.settings.userAgentString = request.header("User-Agent")
|
||||
view.webViewClient = object : WebViewClientCompat() {
|
||||
|
||||
val cloudflareUrl = HttpUrl.parse("${url.scheme()}://$domain/cdn-cgi/l/chk_jschl")!!
|
||||
.newBuilder()
|
||||
.addQueryParameter("jschl_vc", challenge)
|
||||
.addQueryParameter("pass", pass)
|
||||
.addQueryParameter("s", s)
|
||||
.addQueryParameter("jschl_answer", result)
|
||||
.toString()
|
||||
override fun shouldInterceptRequestCompat(
|
||||
view: WebView,
|
||||
url: String
|
||||
): WebResourceResponse? {
|
||||
val isChallengeResolverUrl = isChallengeResolverUrl(url)
|
||||
if (requestUrl != url && !isChallengeResolverUrl) {
|
||||
return WebResourceResponse("text/plain", "UTF-8", null)
|
||||
}
|
||||
|
||||
val cloudflareHeaders = originalRequest.headers()
|
||||
.newBuilder()
|
||||
.add("Referer", url.toString())
|
||||
.add("Accept", "text/html,application/xhtml+xml,application/xml")
|
||||
.add("Accept-Language", "en")
|
||||
.build()
|
||||
if (isChallengeResolverUrl) {
|
||||
isResolvingChallenge = true
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
return GET(cloudflareUrl, cloudflareHeaders, cache = CacheControl.Builder().build())
|
||||
override fun onPageFinished(view: WebView, url: String) {
|
||||
super.onPageFinished(view, url)
|
||||
if (isResolvingChallenge && url == requestUrl) {
|
||||
setResultAndFinish(true)
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
// atob() is browser API, Using Android's own function. (java.util.Base64 can't be used because of min API level)
|
||||
private const val ATOB_JS = """var atob = function (input) { return b64.decode(input) }"""
|
||||
override fun onReceivedErrorCompat(
|
||||
view: WebView,
|
||||
errorCode: Int,
|
||||
description: String?,
|
||||
failingUrl: String,
|
||||
isMainFrame: Boolean
|
||||
) {
|
||||
if ((errorCode != 503 && requestUrl == failingUrl) ||
|
||||
isChallengeResolverUrl(failingUrl)
|
||||
) {
|
||||
setResultAndFinish(false)
|
||||
}
|
||||
}
|
||||
|
||||
private fun setResultAndFinish(resolved: Boolean) {
|
||||
result = resolved
|
||||
latch.countDown()
|
||||
view.stopLoading()
|
||||
view.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
view.loadUrl(requestUrl, headers)
|
||||
}
|
||||
|
||||
latch.await(12, TimeUnit.SECONDS)
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,11 +2,7 @@ package eu.kanade.tachiyomi.network
|
|||
|
||||
import android.content.Context
|
||||
import android.os.Build
|
||||
import okhttp3.Cache
|
||||
import okhttp3.CipherSuite
|
||||
import okhttp3.ConnectionSpec
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.TlsVersion
|
||||
import okhttp3.*
|
||||
import java.io.File
|
||||
import java.io.IOException
|
||||
import java.net.InetAddress
|
||||
|
|
@ -15,11 +11,7 @@ import java.net.UnknownHostException
|
|||
import java.security.KeyManagementException
|
||||
import java.security.KeyStore
|
||||
import java.security.NoSuchAlgorithmException
|
||||
import javax.net.ssl.SSLContext
|
||||
import javax.net.ssl.SSLSocket
|
||||
import javax.net.ssl.SSLSocketFactory
|
||||
import javax.net.ssl.TrustManagerFactory
|
||||
import javax.net.ssl.X509TrustManager
|
||||
import javax.net.ssl.*
|
||||
|
||||
class NetworkHelper(context: Context) {
|
||||
|
||||
|
|
@ -27,7 +19,7 @@ class NetworkHelper(context: Context) {
|
|||
|
||||
private val cacheSize = 5L * 1024 * 1024 // 5 MiB
|
||||
|
||||
private val cookieManager = PersistentCookieJar(context)
|
||||
val cookieManager = AndroidCookieJar(context)
|
||||
|
||||
val client = OkHttpClient.Builder()
|
||||
.cookieJar(cookieManager)
|
||||
|
|
@ -36,12 +28,9 @@ class NetworkHelper(context: Context) {
|
|||
.build()
|
||||
|
||||
val cloudflareClient = client.newBuilder()
|
||||
.addInterceptor(CloudflareInterceptor())
|
||||
.addInterceptor(CloudflareInterceptor(context))
|
||||
.build()
|
||||
|
||||
val cookies: PersistentCookieStore
|
||||
get() = cookieManager.store
|
||||
|
||||
private fun OkHttpClient.Builder.enableTLS12(): OkHttpClient.Builder {
|
||||
if (Build.VERSION.SDK_INT > Build.VERSION_CODES.KITKAT) {
|
||||
return this
|
||||
|
|
|
|||
|
|
@ -1,19 +0,0 @@
|
|||
package eu.kanade.tachiyomi.network
|
||||
|
||||
import android.content.Context
|
||||
import okhttp3.Cookie
|
||||
import okhttp3.CookieJar
|
||||
import okhttp3.HttpUrl
|
||||
|
||||
class PersistentCookieJar(context: Context) : CookieJar {
|
||||
|
||||
val store = PersistentCookieStore(context)
|
||||
|
||||
override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {
|
||||
store.addAll(url, cookies)
|
||||
}
|
||||
|
||||
override fun loadForRequest(url: HttpUrl): List<Cookie> {
|
||||
return store.get(url)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,78 +0,0 @@
|
|||
package eu.kanade.tachiyomi.network
|
||||
|
||||
import android.content.Context
|
||||
import okhttp3.Cookie
|
||||
import okhttp3.HttpUrl
|
||||
import java.net.URI
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
class PersistentCookieStore(context: Context) {
|
||||
|
||||
private val cookieMap = ConcurrentHashMap<String, List<Cookie>>()
|
||||
private val prefs = context.getSharedPreferences("cookie_store", Context.MODE_PRIVATE)
|
||||
|
||||
init {
|
||||
for ((key, value) in prefs.all) {
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
val cookies = value as? Set<String>
|
||||
if (cookies != null) {
|
||||
try {
|
||||
val url = HttpUrl.parse("http://$key") ?: continue
|
||||
val nonExpiredCookies = cookies.mapNotNull { Cookie.parse(url, it) }
|
||||
.filter { !it.hasExpired() }
|
||||
cookieMap.put(key, nonExpiredCookies)
|
||||
} catch (e: Exception) {
|
||||
// Ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun addAll(url: HttpUrl, cookies: List<Cookie>) {
|
||||
val key = url.uri().host
|
||||
|
||||
// Append or replace the cookies for this domain.
|
||||
val cookiesForDomain = cookieMap[key].orEmpty().toMutableList()
|
||||
for (cookie in cookies) {
|
||||
// Find a cookie with the same name. Replace it if found, otherwise add a new one.
|
||||
val pos = cookiesForDomain.indexOfFirst { it.name() == cookie.name() }
|
||||
if (pos == -1) {
|
||||
cookiesForDomain.add(cookie)
|
||||
} else {
|
||||
cookiesForDomain[pos] = cookie
|
||||
}
|
||||
}
|
||||
cookieMap.put(key, cookiesForDomain)
|
||||
|
||||
// Get cookies to be stored in disk
|
||||
val newValues = cookiesForDomain.asSequence()
|
||||
.filter { it.persistent() && !it.hasExpired() }
|
||||
.map(Cookie::toString)
|
||||
.toSet()
|
||||
|
||||
prefs.edit().putStringSet(key, newValues).apply()
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun removeAll() {
|
||||
prefs.edit().clear().apply()
|
||||
cookieMap.clear()
|
||||
}
|
||||
|
||||
fun remove(uri: URI) {
|
||||
prefs.edit().remove(uri.host).apply()
|
||||
cookieMap.remove(uri.host)
|
||||
}
|
||||
|
||||
fun get(url: HttpUrl) = get(url.uri().host)
|
||||
|
||||
fun get(uri: URI) = get(uri.host)
|
||||
|
||||
private fun get(url: String): List<Cookie> {
|
||||
return cookieMap[url].orEmpty().filter { !it.hasExpired() }
|
||||
}
|
||||
|
||||
private fun Cookie.hasExpired() = System.currentTimeMillis() >= expiresAt()
|
||||
|
||||
}
|
||||
|
|
@ -43,7 +43,7 @@ class SettingsAdvancedController : SettingsController() {
|
|||
titleRes = R.string.pref_clear_cookies
|
||||
|
||||
onClick {
|
||||
network.cookies.removeAll()
|
||||
network.cookieManager.removeAll()
|
||||
activity?.toast(R.string.cookies_cleared)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,83 @@
|
|||
package eu.kanade.tachiyomi.util
|
||||
|
||||
import android.annotation.TargetApi
|
||||
import android.os.Build
|
||||
import android.webkit.*
|
||||
|
||||
@Suppress("OverridingDeprecatedMember")
|
||||
abstract class WebViewClientCompat : WebViewClient() {
|
||||
|
||||
open fun shouldOverrideUrlCompat(view: WebView, url: String): Boolean {
|
||||
return false
|
||||
}
|
||||
|
||||
open fun shouldInterceptRequestCompat(view: WebView, url: String): WebResourceResponse? {
|
||||
return null
|
||||
}
|
||||
|
||||
open fun onReceivedErrorCompat(
|
||||
view: WebView,
|
||||
errorCode: Int,
|
||||
description: String?,
|
||||
failingUrl: String,
|
||||
isMainFrame: Boolean) {
|
||||
|
||||
}
|
||||
|
||||
@TargetApi(Build.VERSION_CODES.N)
|
||||
final override fun shouldOverrideUrlLoading(
|
||||
view: WebView,
|
||||
request: WebResourceRequest
|
||||
): Boolean {
|
||||
return shouldOverrideUrlCompat(view, request.url.toString())
|
||||
}
|
||||
|
||||
final override fun shouldOverrideUrlLoading(view: WebView, url: String): Boolean {
|
||||
return shouldOverrideUrlCompat(view, url)
|
||||
}
|
||||
|
||||
@TargetApi(Build.VERSION_CODES.LOLLIPOP)
|
||||
final override fun shouldInterceptRequest(
|
||||
view: WebView,
|
||||
request: WebResourceRequest
|
||||
): WebResourceResponse? {
|
||||
return shouldInterceptRequestCompat(view, request.url.toString())
|
||||
}
|
||||
|
||||
final override fun shouldInterceptRequest(
|
||||
view: WebView,
|
||||
url: String
|
||||
): WebResourceResponse? {
|
||||
return shouldInterceptRequestCompat(view, url)
|
||||
}
|
||||
|
||||
@TargetApi(Build.VERSION_CODES.M)
|
||||
final override fun onReceivedError(
|
||||
view: WebView,
|
||||
request: WebResourceRequest,
|
||||
error: WebResourceError
|
||||
) {
|
||||
onReceivedErrorCompat(view, error.errorCode, error.description?.toString(),
|
||||
request.url.toString(), request.isForMainFrame)
|
||||
}
|
||||
|
||||
final override fun onReceivedError(
|
||||
view: WebView,
|
||||
errorCode: Int,
|
||||
description: String?,
|
||||
failingUrl: String
|
||||
) {
|
||||
onReceivedErrorCompat(view, errorCode, description, failingUrl, failingUrl == view.url)
|
||||
}
|
||||
|
||||
@TargetApi(Build.VERSION_CODES.M)
|
||||
final override fun onReceivedHttpError(
|
||||
view: WebView,
|
||||
request: WebResourceRequest,
|
||||
error: WebResourceResponse
|
||||
) {
|
||||
onReceivedErrorCompat(view, error.statusCode, error.reasonPhrase, request.url
|
||||
.toString(), request.isForMainFrame)
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in a new issue