android-apps/TikTok-Downloader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #15 from fknives/fix-parsing-#14

Browse source 
Fix parsing #14
This commit is contained in:
Gergely Hegedis 2023-12-14 23:34:33 +02:00 • committed by GitHub
commit 027f4e89d6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 61 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

19
app/src/main/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSource.kt
View file

@ -9,12 +9,15 @@ import org.fnives.tiktokdownloader.data.model.VideoInSavingIntoFile
import org.fnives.tiktokdownloader.data.network.exceptions.CaptchaRequiredException
import org.fnives.tiktokdownloader.data.network.exceptions.NetworkException
import org.fnives.tiktokdownloader.data.network.exceptions.ParsingException
import org.fnives.tiktokdownloader.data.network.parsing.converter.VideoFileUrlConverter
import org.fnives.tiktokdownloader.data.network.parsing.response.VideoFileUrl
import org.fnives.tiktokdownloader.data.network.session.CookieStore
class TikTokDownloadRemoteSource(
private val delayBeforeRequest: Long,
private val service: TikTokRetrofitService,
private val cookieStore: CookieStore
private val cookieStore: CookieStore,
private val videoFileUrlConverter: VideoFileUrlConverter,
) {
@Throws(ParsingException::class, NetworkException::class, CaptchaRequiredException::class)
@ -23,9 +26,17 @@ class TikTokDownloadRemoteSource(
wrapIntoProperException {
delay(delayBeforeRequest) // added just so captcha trigger may not happen
val actualUrl = service.getContentActualUrlAndCookie(videoInPending.url)
Logger.logMessage("actualUrl found = ${actualUrl.url}")
delay(delayBeforeRequest) // added just so captcha trigger may not happen
val videoUrl = service.getVideoUrl(actualUrl.url)
val videoUrl: VideoFileUrl
if (actualUrl.url != null) {
Logger.logMessage("actualUrl found = ${actualUrl.url}")
delay(delayBeforeRequest) // added just so captcha trigger may not happen
videoUrl = service.getVideoUrl(actualUrl.url)
} else {
Logger.logMessage("actualUrl not found. Attempting to parse videoUrl")
videoUrl = videoFileUrlConverter.convertSafely(actualUrl.fullResponse)
}
Logger.logMessage("videoFileUrl found = ${videoUrl.videoFileUrl}")
delay(delayBeforeRequest) // added just so captcha trigger may not happen
val response = service.getVideo(videoUrl.videoFileUrl)

20
app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/converter/ActualVideoPageUrlConverter.kt
View file

@ -10,10 +10,18 @@ class ActualVideoPageUrlConverter(
) : ParsingExceptionThrowingConverter<ActualVideoPageUrl>() {
@Throws(IndexOutOfBoundsException::class, CaptchaRequiredException::class)
override fun convertSafely(responseBody: ResponseBody): ActualVideoPageUrl? =
responseBody.string()
.also(throwIfIsCaptchaResponse::invoke)
.split("rel=\"canonical\" href=\"")[1]
.split("\"")[0]
.let(::ActualVideoPageUrl)
override fun convertSafely(responseBody: ResponseBody): ActualVideoPageUrl {
val responseBodyAsString =responseBody.string()
return try {
val actualVideoPageUrl = responseBodyAsString
.also(throwIfIsCaptchaResponse::invoke)
.split("rel=\"canonical\" href=\"")[1]
.split("\"")[0]
ActualVideoPageUrl(actualVideoPageUrl, responseBodyAsString)
} catch(_: Throwable) {
ActualVideoPageUrl(null, responseBodyAsString)
}
}
}

11
app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/converter/ParsingExceptionThrowingConverter.kt
View file

@ -9,13 +9,20 @@ abstract class ParsingExceptionThrowingConverter<T> : Converter<ResponseBody, T>
@Throws(ParsingException::class, CaptchaRequiredException::class)
final override fun convert(value: ResponseBody): T? =
try {
doActionSafely {
convertSafely(value)
}
@Throws(ParsingException::class, CaptchaRequiredException::class)
fun doActionSafely(action: () -> T): T {
try {
return action()
} catch (captchaRequiredException: CaptchaRequiredException) {
throw captchaRequiredException
} catch (throwable: Throwable) {
throw ParsingException(cause = throwable)
}
}
abstract fun convertSafely(responseBody: ResponseBody): T?
abstract fun convertSafely(responseBody: ResponseBody): T
}

15
app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/converter/VideoFileUrlConverter.kt
View file

@ -3,6 +3,7 @@ package org.fnives.tiktokdownloader.data.network.parsing.converter
import okhttp3.ResponseBody
import org.fnives.tiktokdownloader.Logger
import org.fnives.tiktokdownloader.data.network.exceptions.CaptchaRequiredException
import org.fnives.tiktokdownloader.data.network.exceptions.ParsingException
import org.fnives.tiktokdownloader.data.network.parsing.response.VideoFileUrl
class VideoFileUrlConverter(
@ -10,8 +11,18 @@ class VideoFileUrlConverter(
) : ParsingExceptionThrowingConverter<VideoFileUrl>() {
@Throws(IllegalArgumentException::class, IndexOutOfBoundsException::class, CaptchaRequiredException::class)
override fun convertSafely(responseBody: ResponseBody): VideoFileUrl? {
val html = responseBody.string().also(throwIfIsCaptchaResponse::invoke)
override fun convertSafely(responseBody: ResponseBody): VideoFileUrl {
return convert(responseBody.string())
}
@Throws(ParsingException::class, CaptchaRequiredException::class)
fun convertSafely(responseBody: String): VideoFileUrl {
return doActionSafely { convert(responseBody) }
}
@Throws(IllegalArgumentException::class, IndexOutOfBoundsException::class, CaptchaRequiredException::class)
private fun convert(responseBody: String): VideoFileUrl {
val html = responseBody.also(throwIfIsCaptchaResponse::invoke)
val url = tryToParseDownloadLink(html).also { Logger.logMessage("parsed download link = $it") }
?: tryToParseVideoSrc(html).also { Logger.logMessage("parsed video src = $it") }
?: throw IllegalArgumentException("Couldn't parse url from HTML: $html")

2
app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/converter/VideoResponseConverter.kt
View file

@ -5,6 +5,6 @@ import org.fnives.tiktokdownloader.data.network.parsing.response.VideoResponse
class VideoResponseConverter : ParsingExceptionThrowingConverter<VideoResponse>() {
override fun convertSafely(responseBody: ResponseBody): VideoResponse? =
override fun convertSafely(responseBody: ResponseBody): VideoResponse =
VideoResponse(responseBody.contentType(), responseBody.byteStream())
}

2
app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/response/ActualVideoPageUrl.kt
View file

@ -1,3 +1,3 @@
package org.fnives.tiktokdownloader.data.network.parsing.response
class ActualVideoPageUrl(val url: String)
class ActualVideoPageUrl(val url: String?, val fullResponse: String)

3
app/src/main/java/org/fnives/tiktokdownloader/di/module/NetworkModule.kt
View file

@ -7,6 +7,7 @@ import org.fnives.tiktokdownloader.data.network.TikTokDownloadRemoteSource
import org.fnives.tiktokdownloader.data.network.TikTokRetrofitService
import org.fnives.tiktokdownloader.data.network.parsing.TikTokWebPageConverterFactory
import org.fnives.tiktokdownloader.data.network.parsing.converter.ThrowIfIsCaptchaResponse
import org.fnives.tiktokdownloader.data.network.parsing.converter.VideoFileUrlConverter
import org.fnives.tiktokdownloader.data.network.session.CookieSavingInterceptor
import org.fnives.tiktokdownloader.data.network.session.CookieStore
import retrofit2.Converter
@ -47,5 +48,5 @@ class NetworkModule(private val delayBeforeRequest: Long) {
get() = retrofit.create(TikTokRetrofitService::class.java)
val tikTokDownloadRemoteSource: TikTokDownloadRemoteSource
get() = TikTokDownloadRemoteSource(delayBeforeRequest, tikTokRetrofitService, cookieStore)
get() = TikTokDownloadRemoteSource(delayBeforeRequest, tikTokRetrofitService, cookieStore, VideoFileUrlConverter(throwIfIsCaptchaResponse))
}

10
app/src/test/java/org/fnives/uptodate/TikTokDownloadRemoteSourceUpToDateTest.kt
View file

@ -8,7 +8,6 @@ import org.fnives.tiktokdownloader.di.module.NetworkModule
import org.fnives.tiktokdownloader.helper.getResourceFile
import org.junit.jupiter.api.Assertions
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.Timeout
import java.io.File
@ -38,7 +37,7 @@ class TikTokDownloadRemoteSourceUpToDateTest {
actualFile.delete()
actualFile.createNewFile()
actualFile.deleteOnExit()
val expectedFile = getResourceFile(EXPECTED_FILE_PATH)
val expectedFileOptions = EXPECTED_FILE_PATHS.map{getResourceFile(it)}
actualFile.writeText("")
runBlocking { sut.getVideo(parameter).byteStream }.use { inputStream ->
@ -46,12 +45,15 @@ class TikTokDownloadRemoteSourceUpToDateTest {
inputStream.copyTo(outputStream)
}
}
Assertions.assertTrue(FileUtils.contentEquals(expectedFile, actualFile), "The Downloaded file Is Not Matching the expected")
val doesAnyIsTheSameFile = expectedFileOptions.any { expectedFile->
FileUtils.contentEquals(expectedFile, actualFile)
}
Assertions.assertTrue(doesAnyIsTheSameFile, "The Downloaded file Is Not Matching the expected")
}
companion object {
private const val ACTUAL_FILE_PATH = "actual.mp4"
private const val EXPECTED_FILE_PATH = "video/expected.mp4"
private val EXPECTED_FILE_PATHS = listOf("video/expected_option_1.mp4","video/expected_option_2.mp4")
private const val SUBJECT_VIDEO_URL = "https://vm.tiktok.com/ZSQG7SMf/"
}
}

0
app/src/test/resources/video/expected.mp4 → app/src/test/resources/video/expected_option_1.mp4
View file

BIN
app/src/test/resources/video/expected_option_2.mp4 Normal file
View file

Binary file not shown.