Merge pull request #15 from fknives/fix-parsing-#14

Fix parsing #14

app/src/main/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSource.kt

@@ -9,12 +9,15 @@ import org.fnives.tiktokdownloader.data.model.VideoInSavingIntoFile
 import org.fnives.tiktokdownloader.data.network.exceptions.CaptchaRequiredException
 import org.fnives.tiktokdownloader.data.network.exceptions.NetworkException
 import org.fnives.tiktokdownloader.data.network.exceptions.ParsingException
+import org.fnives.tiktokdownloader.data.network.parsing.converter.VideoFileUrlConverter
+import org.fnives.tiktokdownloader.data.network.parsing.response.VideoFileUrl
 import org.fnives.tiktokdownloader.data.network.session.CookieStore
 
 class TikTokDownloadRemoteSource(
     private val delayBeforeRequest: Long,
     private val service: TikTokRetrofitService,
-    private val cookieStore: CookieStore
+    private val cookieStore: CookieStore,
+    private val videoFileUrlConverter: VideoFileUrlConverter,
 ) {
 
     @Throws(ParsingException::class, NetworkException::class, CaptchaRequiredException::class)
@@ -23,9 +26,17 @@ class TikTokDownloadRemoteSource(
         wrapIntoProperException {
             delay(delayBeforeRequest) // added just so captcha trigger may not happen
             val actualUrl = service.getContentActualUrlAndCookie(videoInPending.url)
+            val videoUrl: VideoFileUrl
+            if (actualUrl.url != null) {
                 Logger.logMessage("actualUrl found = ${actualUrl.url}")
                 delay(delayBeforeRequest) // added just so captcha trigger may not happen
-            val videoUrl = service.getVideoUrl(actualUrl.url)
+
+                videoUrl = service.getVideoUrl(actualUrl.url)
+            } else {
+                Logger.logMessage("actualUrl not found. Attempting to parse videoUrl")
+
+                videoUrl = videoFileUrlConverter.convertSafely(actualUrl.fullResponse)
+            }
             Logger.logMessage("videoFileUrl found = ${videoUrl.videoFileUrl}")
             delay(delayBeforeRequest) // added just so captcha trigger may not happen
             val response = service.getVideo(videoUrl.videoFileUrl)

app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/converter/ActualVideoPageUrlConverter.kt

@@ -10,10 +10,18 @@ class ActualVideoPageUrlConverter(
 ) : ParsingExceptionThrowingConverter<ActualVideoPageUrl>() {
 
     @Throws(IndexOutOfBoundsException::class, CaptchaRequiredException::class)
-    override fun convertSafely(responseBody: ResponseBody): ActualVideoPageUrl? =
+    override fun convertSafely(responseBody: ResponseBody): ActualVideoPageUrl {
-        responseBody.string()
+        val responseBodyAsString =responseBody.string()
+        return try {
+            val actualVideoPageUrl = responseBodyAsString
                 .also(throwIfIsCaptchaResponse::invoke)
                 .split("rel=\"canonical\" href=\"")[1]
                 .split("\"")[0]
-            .let(::ActualVideoPageUrl)
+
+            ActualVideoPageUrl(actualVideoPageUrl, responseBodyAsString)
+        } catch(_: Throwable) {
+            ActualVideoPageUrl(null, responseBodyAsString)
+        }
+
+    }
 }

app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/converter/ParsingExceptionThrowingConverter.kt

@@ -9,13 +9,20 @@ abstract class ParsingExceptionThrowingConverter<T> : Converter<ResponseBody, T>
 
     @Throws(ParsingException::class, CaptchaRequiredException::class)
     final override fun convert(value: ResponseBody): T? =
-        try {
+        doActionSafely {
             convertSafely(value)
+        }
+
+    @Throws(ParsingException::class, CaptchaRequiredException::class)
+    fun doActionSafely(action: () -> T): T {
+        try {
+            return action()
         } catch (captchaRequiredException: CaptchaRequiredException) {
             throw captchaRequiredException
         } catch (throwable: Throwable) {
             throw ParsingException(cause = throwable)
         }
+    }
 
-    abstract fun convertSafely(responseBody: ResponseBody): T?
+    abstract fun convertSafely(responseBody: ResponseBody): T
 }

app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/converter/VideoFileUrlConverter.kt

@@ -3,6 +3,7 @@ package org.fnives.tiktokdownloader.data.network.parsing.converter
 import okhttp3.ResponseBody
 import org.fnives.tiktokdownloader.Logger
 import org.fnives.tiktokdownloader.data.network.exceptions.CaptchaRequiredException
+import org.fnives.tiktokdownloader.data.network.exceptions.ParsingException
 import org.fnives.tiktokdownloader.data.network.parsing.response.VideoFileUrl
 
 class VideoFileUrlConverter(
@@ -10,8 +11,18 @@ class VideoFileUrlConverter(
 ) : ParsingExceptionThrowingConverter<VideoFileUrl>() {
 
     @Throws(IllegalArgumentException::class, IndexOutOfBoundsException::class, CaptchaRequiredException::class)
-    override fun convertSafely(responseBody: ResponseBody): VideoFileUrl? {
+    override fun convertSafely(responseBody: ResponseBody): VideoFileUrl {
-        val html = responseBody.string().also(throwIfIsCaptchaResponse::invoke)
+        return convert(responseBody.string())
+    }
+
+    @Throws(ParsingException::class, CaptchaRequiredException::class)
+    fun convertSafely(responseBody: String): VideoFileUrl {
+        return doActionSafely { convert(responseBody) }
+    }
+
+    @Throws(IllegalArgumentException::class, IndexOutOfBoundsException::class, CaptchaRequiredException::class)
+    private fun convert(responseBody: String): VideoFileUrl {
+        val html = responseBody.also(throwIfIsCaptchaResponse::invoke)
         val url = tryToParseDownloadLink(html).also { Logger.logMessage("parsed download link = $it") }
             ?: tryToParseVideoSrc(html).also { Logger.logMessage("parsed video src = $it") }
             ?: throw IllegalArgumentException("Couldn't parse url from HTML: $html")

app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/converter/VideoResponseConverter.kt

@@ -5,6 +5,6 @@ import org.fnives.tiktokdownloader.data.network.parsing.response.VideoResponse
 
 class VideoResponseConverter : ParsingExceptionThrowingConverter<VideoResponse>() {
 
-    override fun convertSafely(responseBody: ResponseBody): VideoResponse? =
+    override fun convertSafely(responseBody: ResponseBody): VideoResponse =
         VideoResponse(responseBody.contentType(), responseBody.byteStream())
 }

app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/response/ActualVideoPageUrl.kt

@@ -1,3 +1,3 @@
 package org.fnives.tiktokdownloader.data.network.parsing.response
 
-class ActualVideoPageUrl(val url: String)
+class ActualVideoPageUrl(val url: String?, val fullResponse: String)

app/src/main/java/org/fnives/tiktokdownloader/di/module/NetworkModule.kt

@@ -7,6 +7,7 @@ import org.fnives.tiktokdownloader.data.network.TikTokDownloadRemoteSource
 import org.fnives.tiktokdownloader.data.network.TikTokRetrofitService
 import org.fnives.tiktokdownloader.data.network.parsing.TikTokWebPageConverterFactory
 import org.fnives.tiktokdownloader.data.network.parsing.converter.ThrowIfIsCaptchaResponse
+import org.fnives.tiktokdownloader.data.network.parsing.converter.VideoFileUrlConverter
 import org.fnives.tiktokdownloader.data.network.session.CookieSavingInterceptor
 import org.fnives.tiktokdownloader.data.network.session.CookieStore
 import retrofit2.Converter
@@ -47,5 +48,5 @@ class NetworkModule(private val delayBeforeRequest: Long) {
         get() = retrofit.create(TikTokRetrofitService::class.java)
 
     val tikTokDownloadRemoteSource: TikTokDownloadRemoteSource
-        get() = TikTokDownloadRemoteSource(delayBeforeRequest, tikTokRetrofitService, cookieStore)
+        get() = TikTokDownloadRemoteSource(delayBeforeRequest, tikTokRetrofitService, cookieStore, VideoFileUrlConverter(throwIfIsCaptchaResponse))
 }

app/src/test/java/org/fnives/uptodate/TikTokDownloadRemoteSourceUpToDateTest.kt

@@ -8,7 +8,6 @@ import org.fnives.tiktokdownloader.di.module.NetworkModule
 import org.fnives.tiktokdownloader.helper.getResourceFile
 import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.BeforeEach
-import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.Timeout
 import java.io.File
@@ -38,7 +37,7 @@ class TikTokDownloadRemoteSourceUpToDateTest {
         actualFile.delete()
         actualFile.createNewFile()
         actualFile.deleteOnExit()
-        val expectedFile = getResourceFile(EXPECTED_FILE_PATH)
+        val expectedFileOptions = EXPECTED_FILE_PATHS.map{getResourceFile(it)}
         actualFile.writeText("")
 
         runBlocking { sut.getVideo(parameter).byteStream }.use { inputStream ->
@@ -46,12 +45,15 @@ class TikTokDownloadRemoteSourceUpToDateTest {
                 inputStream.copyTo(outputStream)
             }
         }
-        Assertions.assertTrue(FileUtils.contentEquals(expectedFile, actualFile), "The Downloaded file Is Not Matching the expected")
+        val doesAnyIsTheSameFile = expectedFileOptions.any { expectedFile->
+            FileUtils.contentEquals(expectedFile, actualFile)
+        }
+        Assertions.assertTrue(doesAnyIsTheSameFile, "The Downloaded file Is Not Matching the expected")
     }
 
     companion object {
         private const val ACTUAL_FILE_PATH = "actual.mp4"
-        private const val EXPECTED_FILE_PATH = "video/expected.mp4"
+        private val EXPECTED_FILE_PATHS = listOf("video/expected_option_1.mp4","video/expected_option_2.mp4")
         private const val SUBJECT_VIDEO_URL = "https://vm.tiktok.com/ZSQG7SMf/"
     }
 }