Scavengetube/index.ts

import parseDuration from "parse-duration";
import humanizeDuration from "humanize-duration";
import { createLogger, format, transports } from "winston";
import { InvalidArgumentError, Command } from "@commander-js/extra-typings";
import shell from "shelljs";
import { youtube } from "@googleapis/youtube";
import fs from "fs";

const program = new Command()
  .name("scavengetube")
  .requiredOption(
    "--api-key <API_KEY>",
    "YouTube API key (or specify via SCAVENGETUBE_YOUTUBE_KEY environment variable)",
    (apiKey: string | undefined) => {
      apiKey = apiKey || process.env.SCAVENGETUBE_YOUTUBE_KEY;
      if (!apiKey) {
        throw new InvalidArgumentError("API key is required.");
      }
      return apiKey;
    }
  )
  .requiredOption(
    "--limit <LIMIT>",
    "Download limit (default: 5)",
    (value: string) => {
      const result = parseInt(value, 5);
      if (isNaN(result)) {
        throw new InvalidArgumentError("Invalid limit.");
      }
      return result;
    },
    10
  )
  .requiredOption(
    "--period <PERIOD>",
    "Delay between searches (default: 3h)",
    (value: string) => {
      const result = parseDuration(value);
      if (!result) {
        throw new InvalidArgumentError("Invalid duration.");
      }
      return result;
    },
    3 * 60 * 60 * 1000
  )
  .requiredOption(
    "--output <DIRECTORY>",
    "Output directory",
    (directory: string) => {
      try {
        fs.accessSync(directory, fs.constants.W_OK);
        return directory;
      } catch (err) {
        throw new InvalidArgumentError(
          `Directory ${directory} is not writable.`
        );
      }
    },
    "out"
  )
  .requiredOption(
    "--state-file <FILE>",
    "State file",
    (file: string) => {
      try {
        fs.accessSync(file, fs.constants.W_OK);
        return file;
      } catch (err) {
        throw new InvalidArgumentError(`File ${file} is not writable.`);
      }
    },
    "state.json"
  )
  .option("--log <LOG_LEVEL>", 'Log level, default "info"', "info");

program.parse();

const options = program.opts();

const logger = createLogger({
  format: format.combine(
    format.timestamp(),
    format.errors({ stack: true }),
    format.splat(),
    format.colorize(),
    format.padLevels(),
    format.printf(({ level, message, timestamp }) => {
      return `${timestamp} | ${level}: ${message}`;
    })
  ),
  transports: [new transports.Console({ level: options.log })],
});

logger.info("Starting scavengetube...");

const ytdlPath = ["yt-dlp", "yt-dl"]
  .map((bin) => shell.which(bin))
  .find(Boolean);
if (!ytdlPath) {
  throw new Error("Couldn't find yt-dlp or yt-dl binary in $PATH.");
}
logger.debug("Using %s as yt-dlp binary.", ytdlPath.toString());

const ytApi = youtube({
  version: "v3",
  auth: options.apiKey,
});

let state = {
  downloaded: [] as string[],
};

if (fs.existsSync(options.stateFile)) {
  logger.info("Loading state from %s", options.stateFile);
  state = JSON.parse(fs.readFileSync(options.stateFile, "utf-8"));
  logger.debug("Loaded state: %j", state);
} else {
  logger.info("No state file found, starting from scratch.");
}

if (!fs.existsSync(options.output)) {
  logger.info("Creating output directory %s", options.output);
  fs.mkdirSync(options.output);
}

while (true) {
  const now = new Date();

  const search = getSearch();
  logger.info("Searching for %s", search);
  try {
    const response = await ytApi.search.list({
      part: ["snippet"],
      q: search,
      maxResults: 50,
      safeSearch: "none",
      order: "date",
    });

    const videos = (response.data.items || []).map((item) => ({
      id: item.id?.videoId,
      title: item.snippet?.title,
      channel: item.snippet?.channelTitle,
      description: item.snippet?.description,
      publishedAt: item.snippet?.publishedAt,
    }));

    logger.debug("Found %d videos...", videos.length);
    logger.debug("%j", videos);

    if (videos.length === 0) {
      logger.warn("Got no videos!");
    }

    let downloaded = 0;
    for (const video of videos) {
      if (!video.id) {
        logger.warn("Skipping %s, no video ID?!", video);
        continue;
      }
      if (state.downloaded.includes(video.id)) {
        logger.debug("Skipping %s, already downloaded", video.id);
        continue;
      }

      logger.info(
        "Downloading %s (%s) from %s by %s",
        video.id,
        video.title,
        video.publishedAt,
        video.channel
      );

      const cmdline = `${ytdlPath} --no-playlist --no-warnings --write-info-json --output "${
        options.output
      }/${new Date().getTime()}_%(title)s_%(id)s.%(ext)s" https://www.youtube.com/watch?v=${
        video.id
      }`;

      logger.debug("Executing `%s`", cmdline);
      const result = shell.exec(cmdline, { silent: true });
      if (result.code !== 0) {
        logger.error("Failed to download %s: %s", video.id, result.stderr);
        continue;
      }

      state.downloaded.push(video.id);
      downloaded++;
      logger.info("Downloaded %s", video.id);

      if (downloaded >= options.limit) {
        logger.info("Download limit reached, stopping search");
        break;
      }
    }
  } catch (err) {
    logger.error(err);
  }

  fs.writeFileSync(options.stateFile, JSON.stringify(state));
  logger.debug("Saved state to %s", options.stateFile);

  const delay = now.getTime() + options.period - Date.now();
  logger.info("Waiting for %s", humanizeDuration(delay));
  await new Promise((resolve) => setTimeout(resolve, delay));
}

function getSearch(): string {
  var prefix = pick(["DSC", "MOV", "IMG", "100", "MVI"]);
  var separator = pick([" ", "_", ""]);
  var numbers = String(Math.floor(Math.random() * 9999)).padStart(4, "0");
  return `${prefix}${separator}${numbers}`;
}

function pick<T>(array: T[]): T {
  return array[Math.floor(Math.random() * array.length)];
}