From 0d5e1a73d2c8cae86a4e928fe379dfc7da4881e4 Mon Sep 17 00:00:00 2001 From: Gergely Hegedus Date: Thu, 21 Apr 2022 14:18:10 +0300 Subject: [PATCH] Fix issue#2 by replacing \u0026 with & character as the url is encoded --- .../org/fnives/tiktokdownloader/Logger.kt | 12 ++++++++++ .../network/TikTokDownloadRemoteSource.kt | 3 +++ .../converter/VideoFileUrlConverter.kt | 19 +++++++++++---- .../network/TikTokDownloadRemoteSourceTest.kt | 24 ++++++++++++++----- .../TikTokDownloadRemoteSourceUpToDateTest.kt | 1 + 5 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 app/src/main/java/org/fnives/tiktokdownloader/Logger.kt diff --git a/app/src/main/java/org/fnives/tiktokdownloader/Logger.kt b/app/src/main/java/org/fnives/tiktokdownloader/Logger.kt new file mode 100644 index 0000000..7facc18 --- /dev/null +++ b/app/src/main/java/org/fnives/tiktokdownloader/Logger.kt @@ -0,0 +1,12 @@ +package org.fnives.tiktokdownloader + +object Logger { + + private const val TAG = "TTDTag" + + fun logMessage(message: String) { + if (BuildConfig.DEBUG) { + System.err.println("TTDTag $message") + } + } +} diff --git a/app/src/main/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSource.kt b/app/src/main/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSource.kt index 4e15411..01e754c 100644 --- a/app/src/main/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSource.kt +++ b/app/src/main/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSource.kt @@ -3,6 +3,7 @@ package org.fnives.tiktokdownloader.data.network import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay import kotlinx.coroutines.withContext +import org.fnives.tiktokdownloader.Logger import org.fnives.tiktokdownloader.data.model.VideoInPending import org.fnives.tiktokdownloader.data.model.VideoInSavingIntoFile import org.fnives.tiktokdownloader.data.network.exceptions.CaptchaRequiredException @@ -22,8 +23,10 @@ class TikTokDownloadRemoteSource( wrapIntoProperException { delay(delayBeforeRequest) // added just so captcha trigger may not happen val actualUrl = service.getContentActualUrlAndCookie(videoInPending.url) + Logger.logMessage("actualUrl found = ${actualUrl.url}") delay(delayBeforeRequest) // added just so captcha trigger may not happen val videoUrl = service.getVideoUrl(actualUrl.url) + Logger.logMessage("videoFileUrl found = ${videoUrl.videoFileUrl}") delay(delayBeforeRequest) // added just so captcha trigger may not happen val response = service.getVideo(videoUrl.videoFileUrl) diff --git a/app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/converter/VideoFileUrlConverter.kt b/app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/converter/VideoFileUrlConverter.kt index 7bfb433..8be5d50 100644 --- a/app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/converter/VideoFileUrlConverter.kt +++ b/app/src/main/java/org/fnives/tiktokdownloader/data/network/parsing/converter/VideoFileUrlConverter.kt @@ -1,9 +1,9 @@ package org.fnives.tiktokdownloader.data.network.parsing.converter import okhttp3.ResponseBody +import org.fnives.tiktokdownloader.Logger import org.fnives.tiktokdownloader.data.network.exceptions.CaptchaRequiredException import org.fnives.tiktokdownloader.data.network.parsing.response.VideoFileUrl -import kotlin.jvm.Throws class VideoFileUrlConverter( private val throwIfIsCaptchaResponse: ThrowIfIsCaptchaResponse @@ -12,8 +12,8 @@ class VideoFileUrlConverter( @Throws(IllegalArgumentException::class, IndexOutOfBoundsException::class, CaptchaRequiredException::class) override fun convertSafely(responseBody: ResponseBody): VideoFileUrl? { val html = responseBody.string().also(throwIfIsCaptchaResponse::invoke) - val url = tryToParseDownloadLink(html) - ?: tryToParseVideoSrc(html) + val url = tryToParseDownloadLink(html).also { Logger.logMessage("parsed download link = $it") } + ?: tryToParseVideoSrc(html).also { Logger.logMessage("parsed video src = $it") } ?: throw IllegalArgumentException("Couldn't parse url from HTML: $html") return VideoFileUrl(url) @@ -26,7 +26,7 @@ class VideoFileUrlConverter( html.split("\"playAddr\"")[1] .dropWhile { it != '\"' }.drop(1) .takeWhile { it != '\"' } - .replace("\\u0026", "&") + .urlCharacterReplacements() } else { null } @@ -39,10 +39,19 @@ class VideoFileUrlConverter( .dropWhile { it != '=' } .dropWhile { it != '\"' }.drop(1) .takeWhile { it != '\"' } - .replace("\\u0026", "&") + .urlCharacterReplacements() } else { null } + private val replacements = mutableMapOf( + "\\u002F" to "/", + "\\u0026" to "&" + ) + + private fun String.urlCharacterReplacements(): String = + replacements.entries.fold(this) { result, toReplaceEntry -> + result.replace(toReplaceEntry.key, toReplaceEntry.value) + } } } \ No newline at end of file diff --git a/app/src/test/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSourceTest.kt b/app/src/test/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSourceTest.kt index 567d25d..c60cb2c 100644 --- a/app/src/test/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSourceTest.kt +++ b/app/src/test/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSourceTest.kt @@ -128,14 +128,19 @@ class TikTokDownloadRemoteSourceTest { mockWebServer.enqueue(MockResponse().setResponseCode(200).setHeader("Content-Type", "video/mp4").setBody("banan")) val videoInPending = VideoInPending("alma", TEST_URL) - val response = sut.getVideo(videoInPending) Assertions.assertEquals(expectedId, response.id) Assertions.assertEquals(expectedUrl, response.url) Assertions.assertEquals(expectedContentType, response.contentType) Assertions.assertEquals("banan", response.byteStream.reader().readText()) + + mockWebServer.takeRequest() + mockWebServer.takeRequest() + val videoRequest = mockWebServer.takeRequest() + Assertions.assertEquals("http://localhost:8080/?a=a&b=b&c=c", videoRequest.requestUrl?.toUri()?.toString()) } + @Test fun GIVEN_proper_responses_as_variant2_THEN_parsed_properly() = runBlocking { val expectedId = "e-alma" @@ -154,6 +159,10 @@ class TikTokDownloadRemoteSourceTest { Assertions.assertEquals(expectedUrl, response.url) Assertions.assertEquals(expectedContentType, response.contentType) Assertions.assertEquals("a-banan", response.byteStream.reader().readText()) + mockWebServer.takeRequest() + mockWebServer.takeRequest() + val videoRequest = mockWebServer.takeRequest() + Assertions.assertEquals("http://localhost:8080/?a=a&b=b&c=c", videoRequest.requestUrl?.toUri()?.toString()) } @Test @@ -259,31 +268,34 @@ class TikTokDownloadRemoteSourceTest { private const val MAIN_PAGE_VARIANT_2_RESPONSE = "response/main_page_v1.html" private const val PORT = 8080 private const val TEST_URL = "http://127.0.0.1:$PORT" + private const val SHORTENED_TEST_URL = "http://127.0.0.1:$PORT" + private const val VIDEO_FILE_TEST_URL = "http:\\u002F/127.0.0.1:$PORT?a=a\\u0026b=b&c=c" + private const val CAPTCHA_TEST_URL = "http://127.0.0.1:$PORT" private fun Any.readResourceFileShortenedUrlResponse() = readResourceFile(SHORTENED_URL_RESPONSE) - .replace("https://www.tiktok.com/@ieclauuu/video/6887614455967010049", TEST_URL) + .replace("https://www.tiktok.com/@ieclauuu/video/6887614455967010049", SHORTENED_TEST_URL) private fun Any.readResourceFileMainPageVariant1Response() = readResourceFile(MAIN_PAGE_VARIANT_1_RESPONSE) .replace( "https://v16-web.tiktok.com/video/tos/alisg/tos-alisg-pve-0037c001/9ddfc12f43b04f6596f9953c9a9ca072/?a=1988\\u0026br=1534\\u0026bt=767\\u0026cr=0\\u0026cs=0\\u0026cv=1\\u0026dr=0\\u0026ds=3\\u0026er=\\u0026expire=1603682739\\u0026l=20201025212533010189074225590A080D\\u0026lr=tiktok_m\\u0026mime_type=video_mp4\\u0026policy=2\\u0026qs=0\\u0026rc=amlxbmV1O291eDMzMzczM0ApZDRoZDQ3Nzw1N2U5Nzs3O2dicW1vL2AxZV5fLS1iMTRzczA2Y2NgYTQ2LmE1Y2E0My46Yw%3D%3D\\u0026signature=cce079fd02e4dde94c1c93cfdbd1d100\\u0026tk=tt_webid_v2\\u0026vl=\\u0026vr=", - TEST_URL + VIDEO_FILE_TEST_URL ) private fun Any.readResourceFileMainPageVariant2Response() = readResourceFile(MAIN_PAGE_VARIANT_2_RESPONSE) .replace( "https://v16-web.tiktok.com/video/tos/alisg/tos-alisg-pve-0037c001/9ddfc12f43b04f6596f9953c9a9ca072/?a=1988\\u0026br=1534\\u0026bt=767\\u0026cr=0\\u0026cs=0\\u0026cv=1\\u0026dr=0\\u0026ds=3\\u0026er=\\u0026expire=1603682739\\u0026l=20201025212533010189074225590A080D\\u0026lr=tiktok_m\\u0026mime_type=video_mp4\\u0026policy=2\\u0026qs=0\\u0026rc=amlxbmV1O291eDMzMzczM0ApZDRoZDQ3Nzw1N2U5Nzs3O2dicW1vL2AxZV5fLS1iMTRzczA2Y2NgYTQ2LmE1Y2E0My46Yw%3D%3D\\u0026signature=cce079fd02e4dde94c1c93cfdbd1d100\\u0026tk=tt_webid_v2\\u0026vl=\\u0026vr=", - TEST_URL + VIDEO_FILE_TEST_URL ) private fun Any.readCaptchaOneResponse() = readResourceFile(CAPTCHA_REQUIRED_RESPONSE_ONE) - .replace("https://www.tiktok.com/@ieclauuu/video/6887614455967010049", TEST_URL) + .replace("https://www.tiktok.com/@ieclauuu/video/6887614455967010049", CAPTCHA_TEST_URL) private fun Any.readCaptchaTwoResponse() = readResourceFile(CAPTCHA_REQUIRED_RESPONSE_TWO) - .replace("https://www.tiktok.com/@ieclauuu/video/6887614455967010049", TEST_URL) + .replace("https://www.tiktok.com/@ieclauuu/video/6887614455967010049", CAPTCHA_TEST_URL) @JvmStatic private fun captchaResponses() = Stream.of( diff --git a/app/src/test/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSourceUpToDateTest.kt b/app/src/test/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSourceUpToDateTest.kt index ea4e177..3a2a03f 100644 --- a/app/src/test/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSourceUpToDateTest.kt +++ b/app/src/test/java/org/fnives/tiktokdownloader/data/network/TikTokDownloadRemoteSourceUpToDateTest.kt @@ -29,6 +29,7 @@ class TikTokDownloadRemoteSourceUpToDateTest { } @Disabled("Can trigger captcha, so only run it separately") + @Timeout(value = 120) @Test fun GIVEN_actualVideo_WHEN_downloading_THEN_the_file_matching_with_the_previously_loaded_video() { val parameter = VideoInPending("123", SUBJECT_VIDEO_URL)