e6* downloader without a memory leak
// ==UserScript==
// @name Red 40
// @description e6* downloader without a memory leak
// @author 1fz54ARh0m8g0KUYzqui
// @license Unlicense
// @version 3.0
// @match https://e621.net/*
// @match https://e6ai.net/*
// @match https://e926.net/*
// @grant GM.getValue
// @grant GM.setValue
// @grant GM_download
// @grant GM_log
// @run-at document-end
// @namespace https://greasyfork.org/users/1518908
// ==/UserScript==
// Verified working on e621ng 25.09.17
// A constructed object cannot be returned from an async IIFE
// This might be the only way to do this
// There is a race condition here
const configuration = {
maximumAttempts: 1,
clientName: "",
rememberHashes: false,
userAgentCompliant: false,
getHashes: async () => {
try {
const retrieved = JSON.parse(GM.getValue("hashes", []));
if (!(retrieved instanceof Array)) {
return [];
}
return retrieved;
} catch {
return [];
}
},
setHashes: async (hashArray) => {
if (!(hashArray instanceof Array)) {
return new TypeError("Array required");
}
await GM.setValue("hashes", JSON.stringify(hashArray));
}
};
// Page data
const currentPage = document.querySelector("nav.pagination.numbered")?.dataset.current | 0;
const finalPage = (document.querySelector("nav.pagination.numbered")?.dataset.total ?? currentPage) | 0;
const hasPosts = !!(document.querySelector("section.posts-container"));
// Locking system modified from
// https://medium.com/@chris_marois/asynchronous-locks-in-modern-javascript-8142c877baf
const metadata = {
unlock: () => {},
lock: () => {
this.promise = new Promise(resolve => this.unlock = resolve);
},
promise: Promise.resolve(),
data: []
};
// Heavy checks since the user can put invalid data
(async function() {
const maximumAttempts = Math.max((await GM.getValue("maximumAttempts", 3)) | 0, 1);
const clientName = String(await GM.getValue("clientName", "e129"));
const rememberHashes = !!(await GM.getValue("rememberHashes", false));
const userAgentCompliant = !!(await GM.getValue("userAgentCompliant", true));
configuration.maximumAttempts = maximumAttempts;
configuration.clientName = clientName;
configuration.rememberHashes = rememberHashes;
configuration.userAgentCompliant = (userAgentCompliant && clientName.length !== 0);
})();
// Having a reusable attempts function was slow and broke
// No point if there's only 2 attemptable functions
// FireMonkey has issues with this, that sucks
async function _download(url, currentAttempt) {
url = String(url);
currentAttempt = Math.max(currentAttempt|0, 1);
// Guess the file name
let fileName = new URL(url).pathname.split("/").pop();
// If the file name is invalid, just go with no extension
if (!/^[0-9A-Za-z.]+$/.test(fileName)) {
fileName = `download@${Date.now()}`;
}
GM_log(`Downloading ${fileName}.`);
const downloadPromise = new Promise((resolve, reject) => {
GM_download({
url: url,
name: fileName,
saveAs: false,
conflictAction: "uniquify",
onerror: (error) => reject(error),
onload: (response) => resolve(response)
});
});
await downloadPromise.catch(async () => {
GM_log(`Failed to download ${fileName}. (Attempt ${currentAttempt}/${configuration.maximumAttempts})`);
if (currentAttempt >= configuration.maximumAttempts) {
return;
}
// Exponential backoff
await new Promise(resolve => setTimeout(resolve, 1000 * Math.pow(2, currentAttempt)));
await _download(url, currentAttempt+1);
});
await downloadPromise.then(() => GM_log(`Downloaded ${fileName}.`));
}
// A wrapper for _download
async function _downloadList(urlList) {
if (!(urlList instanceof Array)) {
return new TypeError("Array required");
}
for (const url of urlList) {
await _download(url, 1);
}
}
async function _fetch(url, currentAttempt) {
url = String(url);
currentAttempt = Math.max(currentAttempt | 0, 1);
let response = await fetch(url);
if (!response.ok) {
GM_log(`Failed to fetch ${url} (Attempt ${currentAttempt}/${configuration.maximumAttempts})`);
if (currentAttempt >= configuration.maximumAttempts) {
return undefined;
}
// Hopefully this frees memory
response = undefined;
// Exponential backoff
await new Promise(resolve => setTimeout(resolve, 1000 * Math.pow(2, currentAttempt)));
return await _fetch(url, currentAttempt+1);
}
response = await response.text();
// This has to be on the same line, I don't know why
const parser = new DOMParser().parseFromString(response, "text/html");
return parser;
}
async function _getMetadata() {
if (!hasPosts) {
return;
}
await metadata.promise;
if (metadata.data.length !== 0) {
return;
}
metadata.lock();
if (finalPage === 1) {
GM_log("Fetching 1 page.");
} else {
GM_log(`Fetching ${(finalPage+1)-currentPage} pages.`);
}
let pagesSkipped = 0;
// The other parameters are unknown, so only replace the page numbers
// A URL is needed, so provide one if there is only one page
const baseURL = new URL(document.querySelector("a.page.last")?.href ?? "https://example.com");
const baseURLParameters = new URLSearchParams(baseURL.search);
if (configuration.userAgentCompliant) {
baseURLParameters.set("_client", String(configuration.clientName));
}
// Keep only one of following pages' DOMs active
for (let i = currentPage; i <= finalPage; i++) {
let parser;
if (i === currentPage) {
parser = document;
} else {
// This should never happen
if (baseURL.hostname === "example.com") {
pagesSkipped++;
continue;
}
baseURLParameters.set("page", i);
baseURL.search = baseURLParameters.toString();
// Fetch isn't async, it just returns a promise
parser = await _fetch(baseURL.toString());
if (parser === undefined) {
pagesSkipped++;
continue;
}
}
const pagePosts = parser.querySelector("section.posts-container")?.querySelectorAll("article");
// There is a chance that a human verification page is returned, nothing can be done
if (pagePosts === null) {
pagesSkipped++;
continue;
}
for (const post of pagePosts) {
// Tags are split by spaces, so split here
const tags = post.dataset.tags.split(' ');
const postData = {
id: post.dataset.id | 0,
extension: post.dataset.fileExt,
md5: post.dataset.md5,
tags: tags.map((e) => e.replaceAll('_', ' ')),
url: post.dataset.fileUrl
};
// Race condition possible, oh well
metadata.data.push(postData);
}
}
metadata.unlock();
if (pagesSkipped === 1) {
GM_log("1 page skipped.");
} else if (pagesSkipped !== 0) {
GM_log(`${pagesSkipped} pages skipped.`);
}
}
const metadataExportButton = document.createElement("a");
metadataExportButton.onclick = async () => {
if (metadataExportButton.disabled) {
return;
}
metadataExportButton.disabled = true;
let jsonData;
if (hasPosts) {
await _getMetadata();
jsonData = JSON.stringify(metadata.data);
} else {
// This data could be cached instead of computed, but what are the benefits?
// There must be at least 1 tag on a post
const tagList = document.querySelector("section#tag-list").querySelectorAll("span.tag-list-name");
const tagObject = {
tags: []
};
tagList.foreach((e) => tagObject.tags.push(e.innerText.trim()));
jsonData = JSON.stringify(tagObject);
}
const blob = new Blob([jsonData], {
type: "application/json"
});
const a = document.createElement("a");
a.setAttribute('download', `download@${Date.now()}.json`);
a.setAttribute('href', window.URL.createObjectURL(blob));
a.click();
metadataExportButton.disabled = false;
};
const postDownloadButton = document.createElement("a");
postDownloadButton.onclick = async () => {
if (postDownloadButton.disabled) {
return;
}
postDownloadButton.disabled = true;
await _getMetadata();
const previousHashes = await configuration.getHashes();
const futureHashes = [];
// This pointer will be changed
let urlList1 = [];
for (const e of metadata.data) {
if (configuration.rememberHashes && previousHashes.includes(e.md5)) {
continue;
}
urlList1.push(e.url);
futureHashes.push(e.md5);
}
const fileCount = urlList1.length;
// 0 is weirdly truthy in this case
const urlList2 = urlList1.filter((e, i) => !!(i & 1));
urlList1 = urlList1.filter((e, i) => !(i & 1));
GM_log(`Downloading ${fileCount} files.`);
postDownloadButton.innerText = `Downloading ${fileCount} files...`;
await Promise.allSettled([_downloadList(urlList1), _downloadList(urlList2)]);
GM_log(`Finished downloading ${fileCount} files.`);
if (configuration.rememberHashes) {
await configuration.setHashes(previousHashes.concat(futureHashes));
GM_log("Saved hashes of the download list.");
}
postDownloadButton.disabled = false;
postDownloadButton.innerText = "Download all posts";
};
const secondaryBar = document.querySelector("menu.nav-secondary.desktop");
if (hasPosts || document.querySelector("section#tag-list")) {
const metadataExportItem = document.createElement("li");
metadataExportItem.id = "subnav-metadata-export";
metadataExportButton.id = "subnav-metadata-export-link";
metadataExportButton.innerText = "Export metadata";
metadataExportItem.appendChild(metadataExportButton);
secondaryBar?.appendChild(metadataExportItem);
}
if (hasPosts) {
const postDownloadItem = document.createElement("li");
postDownloadItem.id = "subnav-post-download";
postDownloadButton.id = "subnav-post-download-link";
postDownloadButton.innerText = "Download all posts";
postDownloadItem.appendChild(postDownloadButton);
secondaryBar?.appendChild(postDownloadItem);
}