DownloadCoubs/coub.py

import json
import os
import shutil
import subprocess
import time
import typing as tp
from dataclasses import dataclass

import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from tqdm import tqdm

import settings


@dataclass
class CoubVideo:
    title: str
    video_url: str
    audio_url: str



def get_login():
    return lambda x, y: (x.send_keys(settings.LOGIN), y.send_keys(settings.PASSWORD))


def get_sources():
    options = webdriver.ChromeOptions()
    options.add_argument('log-level=3')
    options.add_argument('--ignore-ssl-errors')
    options.add_argument("--mute-audio")
    browser = webdriver.Chrome(options=options)

    browser.get(settings.COUB_PAGE)
    browser.find_element_by_class_name("header__login-buttons__log-in").click()
    email = browser.find_element_by_css_selector("input[name='email'][type='email']")
    pwd = browser.find_element_by_css_selector("input[name='password'][type='password']")
    WebDriverWait(browser, 50).until(EC.visibility_of_element_located((By.CLASS_NAME, "registration-form__captcha")))
    get_login()(email, pwd)
    pwd.send_keys(Keys.RETURN)
    WebDriverWait(browser, 200).until(EC.visibility_of_element_located((By.CLASS_NAME, "header-channel__avatar")))
    sources = []
    for item in settings.PARSE:
        browser.get(settings.COUB_PAGE + item)
        WebDriverWait(browser, 200).until(EC.visibility_of_element_located((By.CLASS_NAME, "coub__inner")))
        SCROLL_PAUSE_TIME = 2

        last_height = browser.execute_script("return document.body.scrollHeight")

        while True:
            browser.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(SCROLL_PAUSE_TIME)
            sources.append(browser.page_source)
            new_height = browser.execute_script("return document.body.scrollHeight")
            
            if new_height == last_height:
                break
            last_height = new_height

        sources.append(browser.page_source) 
    
    browser.close()
    return sources


def parse_pages(pages: list) -> list:
    links = []
    print("Parsing source pages...")
    for page in tqdm(pages):
        soup = BeautifulSoup(page, 'lxml')
        def get_links(soup) -> tp.Iterable:
            for item in soup.find_all('div', {"class": "coub__inner"}):
                pg = item.find('input', {"class": "viewer__copy__input"})
                yield pg.get('value') if pg is not None else None
            
        links.extend([i for i in get_links(soup) if i is not None])
    
    return links
        

def get_video_info(link: str) -> CoubVideo:
    data = requests.get(link.strip()).text

    def get_json_from_page(page: str) -> dict:
        lines = page.split("\n")
        for line in lines:
            if 'abuses' in line:
                return json.loads(line)

    info = get_json_from_page(data)
    try:
        title =  link.split("/view/")[-1].replace('\n', '') + "_" + "".join(x for x in info.get('title') if (x.isalnum() or x in "._- "))
        video = info['file_versions']['html5']['video']
        video = video.get('higher') or video.get('high')
        audio = info['file_versions']['html5'].get('audio')
        audio = (audio.get('high') or audio.get('med') if audio else None)
        
        return CoubVideo(
            title,
            video.get('url'),
            audio.get('url') if audio else None
            )
    except:
        print("Video can't be downloaded by this shitty script:\n\t" + link)
    

def from_json(data: list[dict]) -> list[CoubVideo]:
    videos = []
    for item in data:
        videos.append(CoubVideo(
            title=item.get('title'),
            video_url=item.get('video_url'),
            audio_url=item.get('audio_url')
        ))
    
    return videos



def to_json(videos: list[CoubVideo]) -> list[dict]:
    result = []
    for item in videos:
        result.append({
            'title': item.title,
            'video_url': item.video_url,
            'audio_url': item.audio_url
        })
    
    return result

def unique(items: list) -> list:
    return list(dict.fromkeys(items))


def get_videos(links: list) -> list[CoubVideo]:
    return [get_video_info(i) for i in tqdm(links)]


def http_get(url: str) -> requests.Response:
    return requests.get(url, stream=True)


def save(data: bytes, name: str, ext: str):
    with open(f"{settings.TEMP_DIR}/{name}.{ext}", "wb") as f:
        f.write(data)

def download_video(video: CoubVideo):
    if os.path.exists(f"{settings.OUT_DIR}/{video.title}.mp4"):
        return

    print(video.title)
    save(http_get(video.video_url).content, video.title, "mp4")
    if video.audio_url:
        save(http_get(video.audio_url).content, video.title, "mp3")
        execute_ffmpeg(video.title)
    else:
        shutil.copy(f"{settings.TEMP_DIR}/{video.title}.mp4", f"{settings.OUT_DIR}/{video.title}.mp4")

def execute_ffmpeg(path: str):
    proc = subprocess.Popen(["ffmpeg.exe", "-stream_loop", "-1", "-i", f"{settings.TEMP_DIR}/{path}.mp4", "-i", f"{settings.TEMP_DIR}/{path}.mp3", "-shortest", "-map", "0:v:0", "-map", "1:a:0", "-y", f"{settings.OUT_DIR}/{path}.mp4"])
    proc.wait()