Skip to content
Snippets Groups Projects
YoutubeScript.js 465 KiB
Newer Older
Koen's avatar
Koen committed
		this.continuation = itemSection.continuation;
	}

	nextPage() {
		this.context.page = this.context.page + 1;
		if(this.continuation) {
			const continueItems = validateContinuation(()=>
				requestSearchContinuation(this.continuation.token));
			if(continueItems.length > 0) {
				const fakeSectionList = {
					contents: continueItems
				};
				const newItemSection = extractSectionListRenderer_Sections(fakeSectionList, this.context);
				if(newItemSection.channels)
					return new SearchItemSectionPlaylistPager(newItemSection);
			}
		}
		this.hasMore = false;
		this.results = [];
		return this;
	}
}
//#endregion

//#region Requests
function getAuthContextHeaders(useMobile = false, contentType = null) {
	const clientContext = getClientContext(true);
	const result = {
		"Accept-Language": "en-US",
		"x-goog-authuser": "0",
		"x-goog-pageid": clientContext.DELEGATED_SESSION_ID,
		"x-origin": useMobile ? URL_BASE_M : URL_BASE,
		"x-youtube-client-name": useMobile ? "2" : "1",
		"User-Agent": useMobile ? USER_AGENT_TABLET : USER_AGENT_WINDOWS 
	};
	if(contentType)
	    result["Content-Type"] = contentType;

	return result;
}

function requestGuide(pageId) {
	if(!pageId)
		throw new ScriptException("No page id found, invalid authentication?");

	const clientContext = getClientContext(true);
	const body = {
		context: clientContextAuth.INNERTUBE_CONTEXT
	};
	const url = URL_GUIDE + "?key=" + clientContext.INNERTUBE_API_KEY + "&prettyPrint=false"

	const res = http.POST(url, JSON.stringify(body), getAuthContextHeaders(false, "application/json"), true);
	if (res.code != 200) {
		bridge.log("Failed to retrieve subscriptions page.");
		return [];
	}
	const data = JSON.parse(res.body);
	return data;
}
function requestNext(body, useAuth = false, useMobile = false) {
Koen's avatar
Koen committed
	const clientContext = getClientContext(useAuth);
	if(!clientContext || !clientContext.INNERTUBE_CONTEXT || !clientContext.INNERTUBE_API_KEY)
		throw new ScriptException("Missing client context");
	body.context = clientContext.INNERTUBE_CONTEXT;
	const baseUrl = (useMobile) ? URL_NEXT_MOBILE : URL_NEXT;
	const url = baseUrl + "?key=" + clientContext.INNERTUBE_API_KEY + "&prettyPrint=false";
	let headers = (!bridge.isLoggedIn() && useAuth) ? {} : getAuthContextHeaders(useMobile);
	headers["Content-Type"] = "application/json";
	if(useMobile) {
		headers["User-Agent"] = USER_AGENT_TABLET;
	}
	if(useAuth) {
		headers["x-goog-authuser"] = clientContext.SESSION_INDEX ?? "0";
	}
	const resp = http.POST(url, JSON.stringify(body), headers, useAuth);
Koen's avatar
Koen committed
	if(!resp.isOk) {
		log("Fail Url: " + url + "\nFail Body:\n" + JSON.stringify(body));
		throw new ScriptException("Failed to next [" + resp.code + "]");
	}
	return JSON.parse(resp.body);
}
function requestBrowse(body, useMobile = false, useAuth = false, attempt = 0) {
Koen's avatar
Koen committed
	const clientContext = getClientContext(useAuth);
	if(!clientContext || !clientContext.INNERTUBE_CONTEXT || !clientContext.INNERTUBE_API_KEY)
		throw new ScriptException("Missing client context");
	body.context = clientContext.INNERTUBE_CONTEXT;

	let headers = !bridge.isLoggedIn() ? {} : getAuthContextHeaders(useMobile);
	if(useMobile)
		headers["User-Agent"] = USER_AGENT_TABLET;
	headers["Content-Type"] = "application/json";
 
	const baseUrl = !useMobile ? URL_BROWSE : URL_BROWSE_MOBILE;
	const url = baseUrl + "?key=" + clientContext.INNERTUBE_API_KEY + "&prettyPrint=false";
	const resp = http.POST(url, JSON.stringify(body), headers, useAuth);
	if(!resp.isOk) {
	    if((resp.code == 408 || resp.code == 500) && attempt < 1) {
	        return requestBrowse(body, useMobile, useAuth, attempt + 1);
	    }

Koen's avatar
Koen committed
		log("Fail Url: " + url + "\nFail Body:\n" + JSON.stringify(body));

		if(resp.code != 500 || !bridge.isLoggedIn())
		    throw new ScriptException("Failed to browse [" + resp.code + "]");
		else {
		    throw new ScriptLoginRequiredException("Failed to browse [" + resp.code + "]\nLogin might have expired, try logging in again");
		}
Koen's avatar
Koen committed
	}
	return JSON.parse(resp.body);
}
function requestSearch(query, useAuth = false, params = null) {
	const clientContext = getClientContext(useAuth);
	if(!clientContext || !clientContext.INNERTUBE_CONTEXT || !clientContext.INNERTUBE_API_KEY)
		throw new ScriptException("Missing client context");

	const body = {
		context: clientContext.INNERTUBE_CONTEXT,
		query: query
	};
	if(params)
	    body.params = params;
	
	const resp = http.POST(URL_SEARCH + "?key=" + clientContext.INNERTUBE_API_KEY + "&prettyPrint=false",
		JSON.stringify(body), {
			"User-Agent": USER_AGENT_WINDOWS,
			"Content-Type": "application/json"
		}, useAuth);
	if(!resp.isOk) throw new ScriptException("Failed to search [" + resp.code + "]");

	return JSON.parse(resp.body);
}
function requestSearchContinuation(continuation, useAuth = false) {
	const clientContext = getClientContext(useAuth);
	if(!clientContext || !clientContext.INNERTUBE_CONTEXT || !clientContext.INNERTUBE_API_KEY)
		throw new ScriptException("Missing client context");

	const body = {
		context: clientContext.INNERTUBE_CONTEXT,
		continuation: continuation
	};
	
	const resp = http.POST(URL_SEARCH + "?key=" + clientContext.INNERTUBE_API_KEY + "&prettyPrint=false",
		JSON.stringify(body), {
			"Content-Type": "application/json"
		}, useAuth);
	if(!resp.isOk) throw new ScriptException("Failed to search [" + resp.code + "]");

	return JSON.parse(resp.body);
}

Kelvin's avatar
Kelvin committed
function getRequestHeaders(additionalHeaders) {
	const headers = additionalHeaders ?? {};
	return Object.assign(headers, {"Accept-Language": "en-US"});
}
Koen's avatar
Koen committed
function requestPage(url, headers, useAuth = false) {
Kelvin's avatar
Kelvin committed
	const resp = http.GET(url, getRequestHeaders(headers), useAuth);
Koen's avatar
Koen committed
	throwIfCaptcha(resp);

Koen's avatar
Koen committed
	if(resp.isOk)
		return resp.body;
	else throw new ScriptException("Failed to request page [" + resp.code + "]");
}
function requestInitialData(url, useMobile = false, useAuth = false) {
Kelvin's avatar
Kelvin committed
	let headers = {"Accept-Language": "en-US", "Cookie": "PREF=hl=en&gl=US" };
Koen's avatar
Koen committed
	if(useMobile)
		headers["User-Agent"] = USER_AGENT_TABLET;

Kelvin's avatar
Kelvin committed

Koen's avatar
Koen committed
	const resp = http.GET(url, headers, useAuth);
Kelvin's avatar
Kelvin committed
	throwIfCaptcha(resp);
Koen's avatar
Koen committed
	if(resp.isOk) {
		let html = resp.body;

		if(html.indexOf("<form action=\"https://consent.youtube.com/save\"") > 0) {
		    log("Consent form required");
		    const consentData = "gl=US&m=0&app=0&pc=yt&continue=" + encodeURIComponent(url) + "&x=6&bl=boq_identityfrontenduiserver_20231017.04_p0&hl=en&src=1&cm=2&set_eom=true";
Kelvin's avatar
Kelvin committed
		    const respConsent = http.POST("https://consent.youtube.com/save", consentData,
		    {
		        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
		        "Accept-Language": "en-US",
		        "Content-Type": "application/x-www-form-urlencoded"
		    }, useAuth);
            throwIfCaptcha(respConsent);
		    if(respConsent.isOk) {
                const body = respConsent.body;
                if(respConsent.body.indexOf("<form action=\"https://consent.youtube.com/save\"") > 0)
                    throw new CriticalException("Failed to refuse Google consent [" + respConsent.code + "]")
                else
                    html = respConsent.body;
		    }
		    else throw new CriticalException("Failed to refuse Google consent [" + resp.code + "]");
		}

Koen's avatar
Koen committed
		const initialData = getInitialData(html);
		return initialData;
	}
Koen's avatar
Koen committed
	else throw new ScriptException("Failed to request page [" + resp.code + "]\n" + url + "\n");
Koen's avatar
Koen committed
}
function requestClientConfig(useMobile = false, useAuth = false) {
	let headers = {

	}
	if(useMobile)
		headers["User-Agent"] = USER_AGENT_TABLET;

	const resp = http.GET(!useMobile ? URL_CONTEXT : URL_CONTEXT_M, headers, useAuth);
Koen's avatar
Koen committed
	if(!resp.isOk) throw new ScriptException("Failed to request context requestClientConfig");
Koen's avatar
Koen committed
	return getClientConfig(resp.body);
}
Kelvin's avatar
Kelvin committed
function requestIOSStreamingData(videoId, batch) {
Koen's avatar
Koen committed
	const body = {
		videoId: videoId,
		cpn: "" + randomString(16),
		contentCheckOk: "true",
		racyCheckOn: "true",
		context: {
			client: {
				"clientName": "IOS",
Kelvin's avatar
Kelvin committed
             	"clientVersion": IOS_APP_VERSION,//"17.31.4",^M
Koen's avatar
Koen committed
				"deviceMake": "Apple",
Kelvin's avatar
Kelvin committed
				"deviceModel": IOS_DEVICE_VERSION,//"iPhone14,5",^M
Koen's avatar
Koen committed
				"platform": "MOBILE",
				"osName": "iOS",
Kelvin's avatar
Kelvin committed
				"osVersion": IOS_OS_VERSION_DETAILED,//"15.6.0.19G71",^M
Koen's avatar
Koen committed
				"hl": langDisplay,
				"gl": langRegion,
			},
			user: {
				"lockedSafetyMode": false
			}
		}
	};
	const headers = {
		"Content-Type": "application/json",
		"User-Agent": USER_AGENT_IOS,
		"X-Goog-Api-Format-Version": "2"
	};

	const token = randomString(12);
	const clientContext = getClientContext(false);
	const url = URL_PLAYER + 
		"?key=" + clientContext.INNERTUBE_API_KEY +
		"&prettyPrint=false" + 
		"&t=" + token +
		"&id=" + videoId

Kelvin's avatar
Kelvin committed
	if(batch) {
		batch.POST(url, JSON.stringify(body), headers, false);
Koen's avatar
Koen committed
		return null;
Kelvin's avatar
Kelvin committed
	}
	else {
		const resp = http.POST(url, JSON.stringify(body), headers, false);
		return resp;
	}
Koen's avatar
Koen committed
}
function requestAndroidStreamingData(videoId) {
	const body = {
		videoId: videoId,
		cpn: "" + randomString(16),
		contentCheckOk: "true",
		racyCheckOn: "true",
		context: {
			client: {
				"clientName": "ANDROID",
				"clientVersion": "17.31.35",
				"platform": "MOBILE",
				"osName": "Android",
				"osVersion": "12",
				"androidSdkVersion": 31,
				"hl": langDisplay,
				"gl": langRegion,
				"params": "8AEB"
			},
			user: {
				"lockedSafetyMode": false
			}
		}
	};
	const headers = {
		"Content-Type": "application/json",
		"User-Agent": USER_AGENT_ANDROID,
		"X-Goog-Api-Format-Version": "2"
	};

	const token = randomString(12);
	const clientContext = getClientContext(false);
	const url = URL_PLAYER + 
		"?key=" + clientContext.INNERTUBE_API_KEY +
		"&prettyPrint=false" + 
		"&t=" + token +
		"&id=" + videoId

	const resp = http.POST(url, JSON.stringify(body), headers, false);
	if(resp.isOk)
		return JSON.parse(resp.body);
	else
		return null;
}
function requestTvHtml5EmbedStreamingData(videoId, sts, withLogin = false) {
Koen's avatar
Koen committed
	const body = {
		videoId: videoId,
		cpn: "" + randomString(16),
		contentCheckOk: "true",
		racyCheckOn: "true",
		playbackContext: {
			contentPlaybackContext: {
				signatureTimestamp: sts,
				referer: "https://www.youtube.com/watch?v=" + videoId
			}
		},
		context: {
			client: {
				"clientName": "TVHTML5_SIMPLY_EMBEDDED_PLAYER",
				"clientVersion": "2.0",
				"clientScreen": "EMBED",
				"platform": "TV",
				"hl": langDisplay,
				"gl": langRegion
			},
			thirdParty: {
				"embedUrl": "https://www.youtube.com/watch?v=" + videoId,
			},
			user: {
				"lockedSafetyMode": false
			}
		}
	};
	const headers = {
		"Content-Type": "application/json",
		"User-Agent": USER_AGENT_TVHTML5_EMBED,
		"X-Goog-Api-Format-Version": "2"
	};

	const token = randomString(12);
	const clientContext = getClientContext(false);
	const url = URL_PLAYER + 
		"?key=" + clientContext.INNERTUBE_API_KEY +
		"&prettyPrint=false" + 
		"&t=" + token +
		"&id=" + videoId

	const resp = http.POST(url, JSON.stringify(body), headers, !!withLogin);
Koen's avatar
Koen committed
	if(resp.isOk)
		return JSON.parse(resp.body);
	else
		return null;
}
//#endregion

//#region Page Extraction
function getInitialData(html, useAuth = false) {
	const clientContext = getClientContext(useAuth);

	//TODO: Fix regex instead of this temporary workaround.
	/*
	const startIndex = html.indexOf("var ytInitialData = ");
	const endIndex = html.indexOf(";</script>", startIndex);
	if(startIndex > 0 && endIndex > 0) {
	    const raw = html.substring(startIndex + 20, endIndex);
	    const initialDataRaw = raw.startsWith("'") && raw.endsWith("'") ?
            decodeHexEncodedString(raw.substring(1, raw.length - 1))
                //TODO: Find proper decoding strat
                .replaceAll("\\\\\"", "\\\"") :
            raw;
		let initialData = null;
		try{
			initialData = JSON.parse(initialDataRaw);
		}
		catch(ex) {
			console.log("Failed to parse initial data: ", initialDataRaw);
			throw ex;
		}
		if(clientContext?.INNERTUBE_CONTEXT && !clientContext.INNERTUBE_CONTEXT.client.visitorData &&
			initialData.responseContext?.webResponseContextExtensionData?.ytConfigData?.visitorData) {
				clientContext.INNERTUBE_CONTEXT.client.visitorData = initialData.responseContext?.webResponseContextExtensionData?.ytConfigData?.visitorData
			log("Found new visitor (auth) data: " + clientContext.INNERTUBE_CONTEXT.client.visitorData);
		}
		return initialData;
	}*/

Koen's avatar
Koen committed
	const match = html.match(REGEX_INITIAL_DATA);
	if(match) {
		const initialDataRaw = match[1].startsWith("'") && match[1].endsWith("'") ?
Koen's avatar
Koen committed
			decodeHexEncodedString(match[1].substring(1, match[1].length - 1))
				//TODO: Find proper decoding strat
				.replaceAll("\\\\\"", "\\\"") : 
			match[1];
		let initialData = null;
Koen's avatar
Koen committed
		try{
			initialData = JSON.parse(initialDataRaw);
		}
		catch(ex) {
			console.log("Failed to parse initial data: ", initialDataRaw);
			throw ex;
		}
		
		
		if(clientContext?.INNERTUBE_CONTEXT && !clientContext.INNERTUBE_CONTEXT.client.visitorData &&
			initialData.responseContext?.webResponseContextExtensionData?.ytConfigData?.visitorData) {
				clientContext.INNERTUBE_CONTEXT.client.visitorData = initialData.responseContext?.webResponseContextExtensionData?.ytConfigData?.visitorData
			log("Found new visitor (auth) data: " + clientContext.INNERTUBE_CONTEXT.client.visitorData);
		}
		return initialData;
	}
	//if(initialData == null)
	//    log(html);

Koen's avatar
Koen committed
	return null;
}
function getInitialPlayerData(html) {
	let match = html.match(REGEX_INITIAL_PLAYER_DATA);
Koen's avatar
Koen committed
	if(match) {
		let initialDataRaw = match[1];
		try {
			return JSON.parse(initialDataRaw);
		}
		catch(ex) {
			//Fallback approach
			match = html.match(REGEX_INITIAL_PLAYER_DATA_FALLBACK);
			if(match) {
				initialDataRaw = match[1];
				return JSON.parse(initialDataRaw);
			}
		}
Koen's avatar
Koen committed
	}
	return null;
}
function getClientConfig(html) {
	const matches = html.matchAll(REGEX_YTCFG);
	let match = null;
	for(let m of matches) {
		if(m && m.length >= 2 && m[1].indexOf("INNERTUBE_CONTEXT") > 0) {
			match = m;
		}
	}

	if(!match) throw new ScriptException("Context structure not found");
	return JSON.parse(match[1]);
}
//#endregion

//#region Top-Level Extraction
/**
 * Extract Subscription channels from a submenu obtained from subscriptionsPage
 * @returns  {PlatformAuthorLink[]} Channels
 */
function extractChannelListSubMenuAvatarRenderer_AuthorLink(renderer) {
	const thumbnail = renderer?.thumbnail?.thumbnails && renderer.thumbnail.thumbnails.length > 0 ?
		renderer.thumbnail.thumbnails[renderer.thumbnail.thumbnails.length - 1] :
		null;
	const name = renderer?.accessibility?.accessibilityData?.label ?
		renderer.accessibility.accessibilityData.label.trim() :
		"";
	const url = renderer?.navigationEndpoint?.browseEndpoint?.canonicalBaseUrl ?
		URL_BASE + renderer.navigationEndpoint.browseEndpoint.canonicalBaseUrl :
		null;
	if(!url || !name)
		return null;
	else
Kelvin's avatar
Kelvin committed
		return new PlatformAuthorLink(new PlatformID(PLATFORM, null, config?.id, PLATFORM_CLAIMTYPE), name, url, thumbnail);
Koen's avatar
Koen committed
}
/**
 * Extract Subscription channels from a submenu obtained from subscriptionsPage
 * @returns  {String[]} Urls
 */
function extractChannelListSubMenuAvatarRenderer_URL(renderer) {
Koen's avatar
Koen committed
	const canonicalUrl = renderer?.navigationEndpoint?.browseEndpoint?.canonicalBaseUrl ?
Koen's avatar
Koen committed
		URL_BASE + renderer.navigationEndpoint.browseEndpoint.canonicalBaseUrl :
		null;
Koen's avatar
Koen committed
	const idUrl = renderer?.navigationEndpoint?.browseEndpoint?.browseId ?
		URL_BASE + "/channel/" + renderer.navigationEndpoint.browseEndpoint.browseId :
		null;
	const url = idUrl ?? canonicalUrl;
Koen's avatar
Koen committed
	if(!url)
		return null;
	else
		return url;
}
/**
 * Extract Subscription channels from a sections[] obtained from guide()
 * @returns {PlatformAuthorLink[]} Channels
 */
function extractGuide_Channels(data) {
	let sections = data.items ?? [];
	let channels = [];

	for(let section of sections) {
		switchKey(section, {
			guideSubscriptionsSectionRenderer(renderer) {
				for(let item of renderer.items) {
					switchKey(item, {
						guideEntryRenderer(guideEntryRenderer) {
							channels.push(extractGuideEntry_AuthorLink(guideEntryRenderer));
						},
						guideCollapsibleEntryRenderer(collapseRenderer) {
							if(collapseRenderer.expandableItems?.length > 0) {
								for(let item of collapseRenderer.expandableItems) {
									switchKey(item, {
										guideEntryRenderer(guideEntryRenderer) {
											channels.push(extractGuideEntry_AuthorLink(guideEntryRenderer));
										}
									})
								}
							}
						}
					});
				}
			}
		});
	}

	return channels;
}
function extractGuideEntry_AuthorLink(guideEntryRenderer) {
	const thumbnail = guideEntryRenderer.thumbnail?.thumbnails?.length > 0 ? 
		guideEntryRenderer.thumbnail.thumbnails[0].url : null;
	const name = guideEntryRenderer.formattedTitle?.simpleText ?? 
		guideEntryRenderer.accessibility?.accessibilityData?.label;
	const url = guideEntryRenderer.navigationEndpoint?.browseEndpoint?.canonicalBaseUrl ?
		URL_BASE + guideEntryRenderer.navigationEndpoint?.browseEndpoint?.canonicalBaseUrl : null;

Kelvin's avatar
Kelvin committed
	return new PlatformAuthorLink(new PlatformID(PLATFORM, null, config.id, PLATFORM_CLAIMTYPE), name, url, thumbnail);
Koen's avatar
Koen committed
}

/**
 * Extract all video results and shelves from a search page's initial data
 * @param data Root-data from search()
 * @param contextData Any context values used to fill out data for resulting objects
 * @returns Object containing videos and shelves
 */
function extractSearch_SearchResults(data, contextData) {
	let searchContents = data.contents?.twoColumnSearchResultsRenderer?.primaryContents?.sectionListRenderer ??
		data.contents?.sectionListRenderer;

	if(searchContents) {
		const results = extractSectionListRenderer_Sections(searchContents, contextData);
		return results;
	}
	return {};
}

/**
 * Extracts a PlatformChannel from a channel page's initial data
 * @param initialData Initial data from a ChannelPage 
 * @returns {PlatformChannel}
 */
function extractChannel_PlatformChannel(initialData, sourceUrl = null) {
	const errorAlerts = initialData?.alerts?.filter(x=>x.alertRenderer?.type == "ERROR") ?? [];
	if(errorAlerts.length > 0){
		throw new UnavailableException(extractText_String(errorAlerts[0].alertRenderer.text));
	}

Kelvin's avatar
Kelvin committed
    if(initialData?.header?.c4TabbedHeaderRenderer) {
        const headerRenderer = initialData?.header?.c4TabbedHeaderRenderer;

        if(IS_TESTING)
            console.log("Initial Data", initialData);

        const thumbnailTargetWidth = 200;
        const thumbnails = headerRenderer.avatar?.thumbnails;
        const thumbnail = (thumbnails && thumbnails.length > 0) ? thumbnails.sort((a,b)=>Math.abs(a.width - thumbnailTargetWidth) - Math.abs(b.width - thumbnailTargetWidth))[0] : { url: "" };
        const banners = headerRenderer.banner?.thumbnails;
        const bannerTargetWidth = 1080;
        const banner = (banners && banners.length > 0) ? banners.sort((a,b)=>Math.abs(a.width - bannerTargetWidth) - Math.abs(b.width - bannerTargetWidth))[0] : { url: "" };

        const idUrl = headerRenderer?.navigationEndpoint?.browseEndpoint?.browseId ?
            URL_BASE + "/channel/" + headerRenderer.navigationEndpoint.browseEndpoint.browseId :
            null;
        const canonicalUrl = headerRenderer?.navigationEndpoint?.browseEndpoint?.canonicalBaseUrl ?
            URL_BASE + headerRenderer.navigationEndpoint.browseEndpoint.canonicalBaseUrl :
            null;

        return new PlatformChannel({
            id: new PlatformID(PLATFORM, headerRenderer.channelId, config.id, PLATFORM_CLAIMTYPE),
            name: headerRenderer.title ?? "",
            thumbnail: thumbnail.url,
            banner: banner.url,
            subscribers: Math.max(0, extractHumanNumber_Integer(extractText_String(headerRenderer.subscriberCountText))),
            description: "",
            url: idUrl,
            urlAlternatives: [idUrl, canonicalUrl],
            links: {}
        });
    }
    else if(initialData?.header?.pageHeaderRenderer) {
        log("New channel model");
        const headerRenderer = initialData?.header?.pageHeaderRenderer;

        if(IS_TESTING)
            console.log("Initial Data (New Model)", initialData);

        const thumbnailTargetWidth = 200;
        const thumbnails = headerRenderer?.content?.pageHeaderViewModel?.image?.decoratedAvatarViewModel?.avatar?.avatarViewModel?.image?.sources;
        const thumbnail = (thumbnails && thumbnails.length > 0) ? thumbnails.sort((a,b)=>Math.abs(a.width - thumbnailTargetWidth) - Math.abs(b.width - thumbnailTargetWidth))[0] : { url: "" };
        const banners = headerRenderer?.content?.pageHeaderViewModel?.banner?.imageBannerViewModel?.image?.sources;
        const bannerTargetWidth = 1080;
        const banner = (banners && banners.length > 0) ? banners.sort((a,b)=>Math.abs(a.width - bannerTargetWidth) - Math.abs(b.width - bannerTargetWidth))[0] : { url: "" };

        const id = initialData?.metadata?.channelMetadataRenderer?.externalId;
        if(!id) {
            log("ID not found in new channel viewmodel:" + JSON.stringify(id, null, "   "));
	        if(bridge.devSubmit) bridge.devSubmit("extractChannel_PlatformChannel - ID Not found in new channel view model", JSON.stringify(initialData));
Kelvin's avatar
Kelvin committed
            throw new ScriptException("ID Not found in new channel view model");
        }
Koen's avatar
Koen committed


Kelvin's avatar
Kelvin committed
        const idUrl = id ?
            URL_BASE + "/channel/" + id:
            null;
        const canonicalUrl = initialData?.metadata?.channelMetadataRenderer?.vanityChannelUrl ?
            initialData?.metadata?.channelMetadataRenderer?.vanityChannelUrl :
            null;

        let subCount = 0;
        const metadataRows = headerRenderer?.content?.pageHeaderViewModel?.metadata?.contentMetadataViewModel?.metadataRows;
        for(let row of metadataRows) {
            const subsStr = row.metadataParts.find(x=>x.text?.content?.indexOf("subscribers") > 0)?.text?.content;
            if(!subsStr)
                continue;
            const subsNum = extractHumanNumber_Integer(extractText_String(subsStr));
            if(!isNaN(subsNum) && subsNum > 0) {
               subCount = subsNum;
                break;
            }
        }
Koen's avatar
Koen committed

Kelvin's avatar
Kelvin committed
        return new PlatformChannel({
            id: new PlatformID(PLATFORM, id, config.id, PLATFORM_CLAIMTYPE),
            name: initialData?.metadata?.channelMetadataRenderer?.title ?? "",
            thumbnail: thumbnail.url,
            banner: banner.url,
            subscribers: Math.max(0, subCount),
            description: initialData?.metadata?.channelMetadataRenderer?.description,
            url: idUrl,
            urlAlternatives: [idUrl, canonicalUrl].filter(x=>x != null),
            links: {}
        });
    }
    else {
        log("Missing header: (" + sourceUrl + ")\n" + JSON.stringify(initialData, null, "   "));
	    if(bridge.devSubmit) bridge.devSubmit("extractChannel_PlatformChannel - No header for " + sourceUrl, JSON.stringify(initialData));
Kelvin's avatar
Kelvin committed
        throw new ScriptException("No header for " + sourceUrl);
    }
Koen's avatar
Koen committed
}
/**
 * Extracts multiple tabs from a page that contains a tab rendering
 * @param initialData Initial data from a page with a TwoColumnBrowseResultsRenderer
 * @param contextData Any context values used to fill out data for resulting objects
 * @returns 
 */
function extractPage_Tabs(initialData, contextData) {
	const content = initialData.contents;
	if(!content) {
	    if(bridge.devSubmit) bridge.devSubmit("extractPage_Tabs - Missing contents", JSON.stringify(initialData));
	    throw new ScriptException("Missing contents");
	}
Koen's avatar
Koen committed

	return switchKey(content, {
		twoColumnBrowseResultsRenderer(renderer) {
			return extractTwoColumnBrowseResultsRenderer_Tabs(renderer, contextData);
		},
		singleColumnBrowseResultsRenderer(renderer) {
			return extractSingleColumnBrowseResultsRenderer_Tabs(renderer, contextData);
		},
		default(name) {
	        if(bridge.devSubmit) bridge.devSubmit("extractPage_Tabs - Unknown renderer type: " + name, JSON.stringify(content));
Koen's avatar
Koen committed
			throw new ScriptException("Unknown renderer type: " + name);
		}
	});
}
//#endregion


//#region Layout Extractors
function extractVideoPage_VideoDetails(initialData, initialPlayerData, contextData, jsUrl, useLogin, useAbr) {
Koen's avatar
Koen committed
	const contents = initialData.contents;
	const contentsContainer = contents.twoColumnWatchNextResults?.results?.results ??
		null;
	if(!contentsContainer || !contentsContainer.contents || !initialPlayerData.videoDetails) return null;

	if (IS_TESTING) {
		console.log("initialData: ", initialData);
		console.log("playerData:", initialPlayerData);
		console.log("streamingData:", initialPlayerData?.streamingData);
	}
	const videoDetails = initialPlayerData.videoDetails;
	const nonce = randomString(16);

	const hlsSource = (initialPlayerData?.streamingData?.hlsManifestUrl) ?
		new HLSSource({
			url: initialPlayerData?.streamingData?.hlsManifestUrl
		}) : null;
	const dashSource = (initialPlayerData?.streamingData?.dashManifestUrl) ?
		new DashSource({
			url: initialPlayerData?.streamingData?.dashManifestUrl
		}) : null;

Kelvin's avatar
Kelvin committed
	const abrStreamingUrl = (initialPlayerData.streamingData.serverAbrStreamingUrl) ? 
		decryptUrlN(initialPlayerData.streamingData.serverAbrStreamingUrl, jsUrl, false) : undefined;
	useAbr = abrStreamingUrl && (!!useAbr || USE_ABR_VIDEOS);
Koen's avatar
Koen committed
	const video = {
		id: new PlatformID(PLATFORM, videoDetails.videoId, config.id),
		name: videoDetails.title,
		thumbnails: new Thumbnails(videoDetails.thumbnail?.thumbnails.map(x=>new Thumbnail(escapeUnicode(x.url), x.height)) ?? []),
		author: new PlatformAuthorLink(new PlatformID(PLATFORM, videoDetails.channelId, config.id, PLATFORM_CLAIMTYPE), videoDetails.author, URL_BASE + "/channel/" + videoDetails.channelId, null, null),
Koen's avatar
Koen committed
		duration: parseInt(videoDetails.lengthSeconds),
		viewCount: parseInt(videoDetails.viewCount),
		url: contextData.url,
		isLive: videoDetails?.isLive ?? false,
		description: videoDetails.shortDescription,
		hls: (videoDetails?.isLive ?? false) ? hlsSource : null,
		dash: (videoDetails?.isLive ?? false) ? dashSource : null,
		live: (videoDetails?.isLive ?? false) ? (hlsSource ?? dashSource) : null,
		video: 
			((!useAbr) ?
				extractAdaptiveFormats_VideoDescriptor(initialPlayerData?.streamingData?.adaptiveFormats, jsUrl, contextData, "") :
				extractABR_VideoDescriptor(initialPlayerData, jsUrl)
			)
			?? new VideoSourceDescriptor([]),
Koen's avatar
Koen committed
		subtitles: initialPlayerData
			.captions
			?.playerCaptionsTracklistRenderer
			?.captionTracks
			?.map(x=>{
				let kind = x.baseUrl.match(REGEX_URL_KIND);
				if(kind)
					kind = kind[1];

				if(!kind || kind == "asr") {
					return {
						name: extractText_String(x.name),
						url: x.baseUrl,
						format: "text/vtt",

						getSubtitles() {
							const subResp = http.GET(x.baseUrl, {});
							if(!subResp.isOk)
								return "";
							const asr = subResp.body;
							let lines = asr.match(REGEX_ASR);
							const newSubs = [];
							let skipped = 0;
							for(let i = 0; i < lines.length; i++) {
								const line = lines[i];
								const lineParsed = /<text .*?start="(.*?)" .*?dur="(.*?)".*?>(.*?)<\/text>/gms.exec(line);

								const start = parseFloat(lineParsed[1]);
								const dur = parseFloat(lineParsed[2]);
								let end = start + dur;
Koen's avatar
Koen committed
								const text = decodeHtml(lineParsed[3]);

								const nextLine = (i + 1 < lines.length) ? lines[i + 1] : null;
								if(nextLine) {
									const lineParsedNext = /<text .*?start="(.*?)" .*?dur="(.*?)".*?>(.*?)<\/text>/gms.exec(nextLine);
									const startNext = parseFloat(lineParsedNext[1]);
									const durNext = parseFloat(lineParsedNext[2]);
									const endNext = startNext + durNext;
									if(startNext && startNext < end)
										end = startNext;
								}

Koen's avatar
Koen committed
								newSubs.push((i - skipped + 1) + "\n" +
									toSRTTime(start, true) + " --> " + toSRTTime(end, true) + "\n" +
									text + "\n");
							}
							console.log(newSubs);
							return "WEBVTT\n\n" + newSubs.join('\n');
						}
					};
				}
				else if(kind == "vtt") {
					return {
						name: extractText_String(x.name),
						url: x.baseUrl,
						format: "text/vtt",
					};
				}
				else return null;
			})?.filter(x=>x != null) ?? []
	};

	//Adds HLS stream if any other format is not yet available, mostly relevant for recently ended livestreams.
Kelvin's avatar
Kelvin committed
	if(video.video.videoSources !== null && video.video.videoSources.length == 0 && initialPlayerData?.streamingData?.hlsManifestUrl)
	    video.video.videoSources.push(new HLSSource({url: initialPlayerData.streamingData.hlsManifestUrl}));


Koen's avatar
Koen committed
	//Add additional/better details
	for(let i = 0; i < contentsContainer.contents.length; i++) {
		const content = contentsContainer.contents[i];
		switchKey(content, {
			videoPrimaryInfoRenderer(renderer) {
				//if(renderer.title?.runs)
				//	video.name = extractString_Runs(renderer.title.runs);
				if(renderer.viewCount?.videoViewCountRenderer?.viewCount?.simpleText)
					video.viewCount = extractFirstNumber_Integer(renderer.viewCount?.videoViewCountRenderer?.viewCount.simpleText)
				else if(renderer.viewCount?.videoViewCountRenderer?.viewCount?.runs) {
					video.viewCount = parseInt(extractFirstNumber_Integer(extractRuns_String(renderer.viewCount?.videoViewCountRenderer?.viewCount?.runs)));
				}
				if(renderer.viewCount?.videoViewCountRenderer?.isLive || renderer.viewCount?.videoViewCountRenderer?.viewCount?.isLive)
					video.isLive = true;
				else
					video.isLive = false;
				

				if(renderer.videoActions?.menuRenderer?.topLevelButtons)
					renderer.videoActions.menuRenderer.topLevelButtons.forEach((button)=>{
						switchKey(button, {
							segmentedLikeDislikeButtonRenderer(renderer) {
								const likeButtonRenderer = renderer?.likeButton?.toggleButtonRenderer;
								if(likeButtonRenderer) {
									const likeTextData = likeButtonRenderer.defaultText;
									if(likeTextData){
										if(likeTextData.accessibility?.accessibilityData?.label)
											video.rating = new RatingLikes(extractFirstNumber_Integer(likeTextData.accessibility.accessibilityData.label));
										else if(likeTextData.simpleText)
											video.rating = new RatingLikes(extractHumanNumber_Integer(likeTextData.simpleText));

									}
								}
Kelvin's avatar
Kelvin committed
							},
							segmentedLikeDislikeButtonViewModel(renderer) {
							    if(IS_TESTING)
							        console.log("Found new likes model:", renderer);
							    let likeButtonViewModel = renderer?.likeButtonViewModel;
							    if(likeButtonViewModel.likeButtonViewModel) //Youtube double nested, not sure if a bug on their end which may be removed
							        likeButtonViewModel = likeButtonViewModel.likeButtonViewModel;
							    let toggleButtonViewModel = likeButtonViewModel?.toggleButtonViewModel;
							    if(toggleButtonViewModel.toggleButtonViewModel) //Youtube double nested, not sure if a bug on their end which may be removed
							        toggleButtonViewModel = toggleButtonViewModel.toggleButtonViewModel;

							    const buttonViewModel = toggleButtonViewModel?.defaultButtonViewModel?.buttonViewModel;
							    if(buttonViewModel?.title) {
							        let num = parseInt(buttonViewModel.title);
							        if(!isNaN(num))
							            video.rating = new RatingLikes(num);
                                    num = extractHumanNumber_Integer(buttonViewModel.title);
                                    if(!isNaN(num) && num >= 0)
                                        video.rating = new RatingLikes(num);
Kelvin's avatar
Kelvin committed
                                    else if(buttonViewModel.title?.toLowerCase() == "like")
                                        video.rating = new RatingLikes(0);
                                    else {
	                                    if(bridge.devSubmit) bridge.devSubmit("extractVideoPage_VideoDetails - Found unknown likes model", JSON.stringify(buttonViewModel));
Kelvin's avatar
Kelvin committed
                                        throw new ScriptException("Found unknown likes model, please report to dev:\n" + JSON.stringify(buttonViewModel.title));
Kelvin's avatar
Kelvin committed
							    }
							    else
							        log("UNKNOWN LIKES MODEL:\n" + JSON.stringify(renderer, null, "   "));
Koen's avatar
Koen committed
							}
						});
					});


				if(!video.datetime || video.datetime <= 0) {
					let date = 0;
					
Koen's avatar
Koen committed
					if (date <= 0 && renderer.relativeDateText?.simpleText)
						date = extractAgoText_Timestamp(renderer.relativeDateText.simpleText);
Koen's avatar
Koen committed
					if(date <= 0 && renderer.dateText?.simpleText)
						date = extractDate_Timestamp(renderer.dateText.simpleText);
Koen's avatar
Koen committed
					video.datetime = date;
				}
			},
			videoSecondaryInfoRenderer(renderer) {
				if(renderer.owner.videoOwnerRenderer)
					video.author = extractVideoOwnerRenderer_AuthorLink(renderer.owner.videoOwnerRenderer);
				if(renderer.description?.runs)
					video.description = extractRuns_Html(renderer.description.runs);
			},
			itemSectionRenderer() {
				//Comments
			}
		});
	}

	const scheduledTime = initialPlayerData?.playabilityStatus?.liveStreamability?.liveStreamabilityRenderer?.offlineSlate?.liveStreamOfflineSlateRenderer?.scheduledStartTime;
	
	if(scheduledTime && !isNaN(scheduledTime))
		video.datetime = parseInt(scheduledTime);

    const result = new PlatformVideoDetails(video);
	if(!useLogin){
		result.getComments = function() {
			return extractTwoColumnWatchNextResultContents_CommentsPager(contextData.url, contentsContainer?.contents, useLogin)
		};
	}
Koen's avatar
Koen committed
    return result;
}

function extractABR_VideoDescriptor(initialPlayerData, jsUrl) {
	
	const abrStreamingUrl = (initialPlayerData.streamingData.serverAbrStreamingUrl) ? 
		decryptUrlN(initialPlayerData.streamingData.serverAbrStreamingUrl, jsUrl, false) : undefined;
	if(!abrStreamingUrl)
		return undefined;

	return new UnMuxVideoSourceDescriptor(
		(initialPlayerData.streamingData.adaptiveFormats
			.filter(x => x.mimeType.startsWith("video/webm"))
			.map(y => {
				const codecs = y.mimeType.substring(y.mimeType.indexOf('codecs=\"') + 8).slice(0, -1);
				const container = y.mimeType.substring(0, y.mimeType.indexOf(';'));
				if (codecs.startsWith("av01"))
					return null; //AV01 is unsupported.

				const duration = parseInt(parseInt(y.approxDurationMs) / 1000) ?? 0;
				if (isNaN(duration))
					return null;
				return new YTABRVideoSource({
					name: "UMP " + y.height + "p" + (y.fps ? y.fps : "") + " " + container,
					url: abrStreamingUrl,
					width: y.width,
					height: y.height,
					duration: (!isNaN(duration)) ? duration : 0,
					container: y.mimeType.substring(0, y.mimeType.indexOf(';')),
					codec: codecs,
					bitrate: y.bitrate,
				}, abrStreamingUrl, y, initialPlayerData.playerConfig.mediaCommonConfig.mediaUstreamerRequestConfig.videoPlaybackUstreamerConfig);
			})).filter(x => x != null),
		//Audio
		(initialPlayerData.streamingData.adaptiveFormats
			.filter(x => x.mimeType.startsWith("audio/webm"))
			.map(y => {
				const codecs = y.mimeType.substring(y.mimeType.indexOf('codecs=\"') + 8).slice(0, -1);
				const container = y.mimeType.substring(0, y.mimeType.indexOf(';'));
				if (codecs.startsWith("av01"))
					return null; //AV01 is unsupported.

				const duration = parseInt(parseInt(y.approxDurationMs) / 1000) ?? 0;
				if (isNaN(duration))
					return null;
				return new YTABRAudioSource({
					name: "UMP " + (y.audioTrack?.displayName ? y.audioTrack.displayName : codecs),
					url: abrStreamingUrl,
					width: y.width,
					height: y.height,
					duration: (!isNaN(duration)) ? duration : 0,
					container: y.mimeType.substring(0, y.mimeType.indexOf(';')),
					codec: codecs,
					bitrate: y.bitrate,
					audioChannels: y.audioChannels,
					language: ytLangIdToLanguage(y.audioTrack?.id)
				}, abrStreamingUrl, y, initialPlayerData.playerConfig.mediaCommonConfig.mediaUstreamerRequestConfig.videoPlaybackUstreamerConfig);
			})).filter(x => x != null)
	);
}

Kelvin's avatar
Kelvin committed
function extractAdaptiveFormats_VideoDescriptor(adaptiveSources, jsUrl, contextData, prefix) {   
	const nonce = randomString(16);   
	return adaptiveSources ? new UnMuxVideoSourceDescriptor(   
			adaptiveSources.filter(x=>x.mimeType.startsWith("video/") && (x.url || x.cipher || x.signatureCipher)).map(y=>{   
Kelvin's avatar
Kelvin committed
					const codecs = y.mimeType.substring(y.mimeType.indexOf('codecs=\"') + 8).slice(0, -1);   
					const container = y.mimeType.substring(0, y.mimeType.indexOf(';'));   
					if(codecs.startsWith("av01"))   
							return null; //AV01 is unsupported.   

					const logItag = y.itag ==  134;   
					if(logItag) {   
							//log(videoDetails.title + " || Format " + container + " - " + y.itag + " - " + y.width);   
							log("Source Parameters:\n" + JSON.stringify({   
									url: y.url,   
									cipher: y.cipher,   
									signatureCipher: y.signatureCipher   
							}, null, "   "));   
					}   
					   
					let url = decryptUrlN(y.url, jsUrl, logItag) ?? decryptUrl(y.cipher, jsUrl, logItag) ?? decryptUrl(y.signatureCipher, jsUrl, logItag);   
					if(url.indexOf("&cpn=") < 0)   
							url = url + "&cpn=" + nonce;   

					const duration = parseInt(parseInt(y.approxDurationMs) / 1000) ?? 0;   
					if(isNaN(duration))   
							return null;   

					if(!y.initRange?.end || !y.indexRange?.end)   
							return null;   

					return new YTVideoSource({   
							name: prefix + y.height + "p" + (y.fps ? y.fps : "") + " " + container,   
							url: url,   
							width: y.width,   
							height: y.height,   
							duration: (!isNaN(duration)) ? duration : 0,   
							container: y.mimeType.substring(0, y.mimeType.indexOf(';')),   
							codec: codecs,   
							bitrate: y.bitrate,   

							itagId: y.itag,   
							initStart: parseInt(y.initRange?.start),   
							initEnd: parseInt(y.initRange?.end),   
							indexStart: parseInt(y.indexRange?.start),   
							indexEnd: parseInt(y.indexRange?.end)   
Kelvin's avatar
Kelvin committed
					}, contextData.url);   
Kelvin's avatar
Kelvin committed
			}).filter(x=>x != null),   
			adaptiveSources.filter(x=>x.mimeType.startsWith("audio/") && (x.url || x.cipher || x.signatureCipher)).map(y=>{   
Kelvin's avatar
Kelvin committed
					const codecs = y.mimeType.substring(y.mimeType.indexOf('codecs=\"') + 8).slice(0, -1);   
					const container = y.mimeType.substring(0, y.mimeType.indexOf(';'));   

					let url = decryptUrlN(y.url, jsUrl) ?? decryptUrl(y.cipher, jsUrl) ?? decryptUrl(y.signatureCipher, jsUrl);