// ==UserScript==
// @name Sadpanda Save/Export All Favorites
// @namespace SaddestPanda
// @description Load all favorites to the page and save it, export as JSON data or just copy the textbox contents. Works on exhentai and e-hentai.
// @match *://exhentai.org/favorites.php*
// @match *://e-hentai.org/favorites.php*
// @homepage https://sleazyfork.org/en/scripts/23406-sadpanda-save-export-all-favorites
// @supportURL https://sleazyfork.org/en/scripts/23406-sadpanda-save-export-all-favorites/feedback
// @version 2.5.7
// @grant GM.getValue
// @grant GM.setValue
// @grant GM.deleteValue
// @grant GM_addValueChangeListener
// @grant GM.registerMenuCommand
// @require https://cdn.jsdelivr.net/npm/gm-webext-pref@0.4.2/dist/GM_webextPref.user.js
// @require https://cdnjs.cloudflare.com/ajax/libs/jquery/3.6.3/jquery.min.js
// @require https://cdnjs.cloudflare.com/ajax/libs/he/1.2.0/he.min.js
// ==/UserScript==
//TODO ditch jquery (only a little left)
//LATER TODO ^^^^redo the whole crawling method (make it async)
//LATER TODO maybe improve the json export functionality using API data like x/favorites-scrape. For now the Extended layout is adequate. Magnet links can be generated using the API data as well.
//Top text box inspiration and its code from http://userscripts-mirror.org/scripts/show/173553. But this one won't have an import feature like that one.
//Using Jquery for jquery purposes and he.js for he.js purposes.
//Using GM_webextPref for the preferences menu. Many thanks to the author "eight" for the library.
//2022.11.05 v2.5.0: Thanks Caca Ductile for your large contributions. Thanks dnsev-h for getTimestamp().
//Use the options menu to change these values.
let inserttoTable = true; //Disables inserting new pages to the table. Set this to false if your browser gets real slow. You can still use the textbox to save your favorites.
let pageTimer = 3000; //Waiting timer for each page in milliseconds. Default is 2500. Making it too short might prevent pages from loading.
let playsilentAudio = false; //Play silent audio while loading the pages. Enable this if you will be using your computer while waiting. It might prevent the tab from going into background mode and stop loading the pages.
//Play a silent audio file to prevent throttling (different files for firefox/chromium)
//Might require autoplay to be turned on for the website
//do not initialize on page load (before user gesture)
class SilentAudio {
constructor() {
this.ctx = new AudioContext();
this.source = this.ctx.createConstantSource();
this.gainNode = this.ctx.createGain();
this.gainNode.gain.value = 0.001; // required to prevent popping on start
this.source.connect(this.gainNode);
this.gainNode.connect(this.ctx.destination);
//suspend context and start the source
this.ctx.suspend();
this.source.start();
}
play() {
this.ctx.resume();
}
pause() {
this.ctx.suspend();
}
}
let silentAudio = null;
const pref = GM_webextPref({
default: {
inserttoTable: true,
playsilentAudio: true,
pageTimer: 3000
},
body: [
{
key: "inserttoTable",
type: "checkbox",
label: "Insert new pages into the current page. Disable this if your browser freezes."
},
{
key: "playsilentAudio",
type: "checkbox",
label: "Play silent audio while loading the pages. This might prevent the tab from going into background mode and stop loading the pages."
},
{
key: "pageTimer",
type: "number",
label: "Time (ms) to load each page. Keep this higher than 3000 especially if you set the thumbnails to load with the page."
}
]
});
let pageCounter = 0,
showExport = false,
totalFavoritesCount = 0,
displayMode = "",
nextURL = "",
titlesBackup = "",
jsonData = [],
statusNode = null,
isExhentai = true;
const globalStyle = `
#exportFav {
width: 900px;
}
#exportStatus {
width: 900px;
height: 168px;
margin: auto;
border: 2px solid gray;
padding: 3px;
resize: both;
overflow: auto;
white-space: pre-wrap;
font-family: monospace;
font-size: 13px;
text-align: left;
}
#exportFavsdiv {
padding: 0 6px 0 7px;
border-bottom: 1px solid #00ffff36;
line-height: 18px;
margin-right: 6px;
}
`;
const exhStyle = `
.greenText {
color: rgb(0, 185, 0);
}
.blueText {
color: rgb(0, 202, 212);
}
.redText {
color: rgb(255, 101, 115);
}
`;
const ehStyle = `
.greenText {
color: rgb(0, 126, 0);
}
.blueText {
color: rgb(0, 88, 202);
}
.redText {
color: rgb(245, 0, 0);
}
`;
$(document).ready(function () {
ready();
});
function ready() {
if (document.querySelector('form[name=favform]').innerText == "No hits found") {
return;
}
//Add global style
addMyStyle("SSEAF_style_global", globalStyle);
//Add theme specific style
if (document.location.host.includes("exhentai")) { //exhentai and onion
isExhentai = true;
addMyStyle("SSEAF_style", exhStyle);
} else {
isExhentai = false;
addMyStyle("SSEAF_style", ehStyle);
}
//Get current display mode
displayMode = document.querySelector("[onchange*='inline_set=dm_'").value; // m:minimal, p:minimal+, l:compact, e:extended, t:thumbnail
let topmenu = document.querySelector("#nb");
topmenu.style.justifyContent = "center";
topmenu.style.maxWidth = "1400px";
$('#nb').append($('<div/>').attr('id', 'exportFavsdiv'));
$('#exportFavsdiv').append($('<a/>').attr('href', 'javascript:void(0)').attr('id', 'exportFavorites').text('Load All Favorites'));
$('#exportFavorites').click(insertNodes);
}
//Count favorites and also generate the category dict
function countAllFavs() {
totalFavoritesCount = 0;
let cats = document.querySelectorAll(".ido > .nosel > div[onclick*='favcat=']");
cats.forEach((cat, index) => {
totalFavoritesCount += parseInt(cat.firstElementChild.textContent);
favoriteCategoryDict[cat.lastElementChild.textContent] = index;
});
}
/* //not needed
function destroyBox() {
document.querySelectorAll('.uselessbr').forEach((node) => {
node.remove();
});
document.querySelector("#exportFav").remove();
document.querySelector("#exportStatus").remove();
}
*/
function appendStatusHTML(htmlString) {
statusNode.insertAdjacentHTML("beforeend", htmlString);
statusNode.scrollTo(0, statusNode.scrollHeight);
}
function addRemoveButton(newbutton) {
if (newbutton) {
$('#exportFav').after("<div>");
$('#exportFav').next().append($('<input/>').attr('type', 'button').attr('value', 'Remove Titles').attr('id', 'removeTitles').attr('style', 'margin-top:5px;min-height:18px;'));
if (displayMode == "e") {
$('#exportFav').next().append($('<input/>').attr('type', 'button').attr('value', 'Download JSON Data').attr('id', 'buttonDownloadJSON').attr('style', 'margin-top:5px;margin-left:5px;min-height:18px;box-shadow: 0px 0px 4px #007bb7;'));
document.querySelector("#buttonDownloadJSON").addEventListener("click", downloadJSON, false);
}
} else {
document.querySelector("#removeTitles").removeEventListener("click", restoreTitles, false);
document.querySelector("#removeTitles").value = "Remove Titles";
}
document.querySelector("#removeTitles").addEventListener("click", removeTitles, false);
}
function addRestoreButton() {
const remTitles = document.querySelector("#removeTitles");
remTitles.removeEventListener("click", removeTitles, false);
remTitles.value = "Restore Titles";
remTitles.addEventListener("click", restoreTitles, false);
}
function removeTitles() {
const exportFav = document.querySelector("#exportFav");
titlesBackup = exportFav.textContent;
exportFav.textContent = exportFav.textContent.replace(/( - .*)/g, "");
addRestoreButton();
}
function restoreTitles() {
document.querySelector("#exportFav").textContent = titlesBackup;
addRemoveButton(0);
}
function getText(thisa) {
let thetext = "";
//all display modes
thetext = /* he.encode( */ thisa.href + " - " + thisa.querySelector(".glink").textContent /* ); */
document.querySelector("#exportFav").append(thetext + '\n');
}
function insertNodes() {
const topMenu = $("#nb");
if (showExport == false) {
//Count favorites and generate favoriteCategoryDict
countAllFavs();
showExport = true;
topMenu.after('<br class="uselessbr"/>');
topMenu.after($('<input/>').attr('type', 'button').attr('value', 'START').attr('id', 'startStopBut').attr('href', 'javascript:void(0)').attr('style', 'margin-left:5px;margin-top:5px;min-height:18px;'));
document.querySelector('#startStopBut').addEventListener("click", getThePagesNew, false);
topMenu.after($('<input/>').attr('type', 'button').attr('value', 'Options').attr('id', 'optionsBut').attr('href', 'javascript:void(0)').attr('style', 'margin-top:5px;min-height:18px;'));
$('#optionsBut').click(pref.openDialog);
topMenu.after('<br class="uselessbr"/>');
topMenu.after('<div readonly id="exportStatus" name="Text2" cols="120" rows="8" />');
statusNode = document.querySelector('#exportStatus');
topMenu.after('<h2 class="uselessbr">Status');
topMenu.after('<textarea readonly id="exportFav" name="Text1" cols="120" rows="7" />');
topMenu.after('<h2 class="uselessbr">Text List of Favorites');
topMenu.after('<br class="uselessbr"/><br class="uselessbr"/>');
appendStatusHTML("Keep the delay between pages higher than 3000 or your pages & thumbnails might not load.\n\n");
appendStatusHTML("❗ If you want to have the best backup: <br><span class='greenText'>1. Use <b>Extended layout</b> so you can export JSON data as well. This works without the insert option.</span><br>2. Set thumbnails to load with the page (in e-h settings). You only need this if you are going to save the page afterwards.\n");
appendStatusHTML("- Leave this tab active while the pages are loading if you want your pages and thumbnails to load correctly. (reason: all browsers throttle background tabs)\n");
appendStatusHTML("- Don't change the page layout in another tab while loading.\n");
appendStatusHTML("- If your browser crashes or completely freezes; disable the insert feature in the options.\n\n");
appendStatusHTML("Press START below to start loading the pages.\n");
//Experimental: this is just so we can have some redundancy.
if (!document.querySelector("#SSEAFjsonData") && displayMode == "e") {
let hiddenDiv = document.createElement("pre");
hiddenDiv.id = "SSEAFjsonData";
hiddenDiv.style.display = "none";
document.body.appendChild(hiddenDiv);
}
} else {
//destroyBox();
//showExport = 0;
return;
}
if (window.location.search.includes("page=") || window.location.search.includes("prev=") || window.location.search.includes("next=") || window.location.search.includes("jump=")) {
appendStatusHTML("(Info: The script does not read previous pages.)\n");
return;
}
}
function getThisPage() {
//LATER cleanup this loop forEach
//LATER call this function from the fetch loop
let trlen = document.querySelectorAll(".itg .glname").length;
for (var j = 0; j < trlen; j++) {
let thislink;
//Extended mode
if (displayMode == "e") {
let row = document.querySelectorAll(".itg > tbody > tr")[j];
thislink = row.querySelector(".glink").closest("a");
//get json data from row
jsonData.push(getJSONfromExtendedRow(row));
} else {
thislink = document.querySelectorAll(".itg .glname a")[j];
}
getText(thislink);
}
//Get nextURL
nextURL = document.querySelector(".searchnav a[href*='next=']")?.href;
}
function getThePagesNew() {
jsonData = [];
let interruptSignal = false;
const startStopButton = document.querySelector('#startStopBut');
startStopButton.removeEventListener("click", getThePagesNew, false);
startStopButton.setAttribute("value", "STOP");
startStopButton.setAttribute("style", "box-shadow: 0px 0px 4px #B7002B;");
startStopButton.blur();
startStopButton.addEventListener("click", () => {
if (confirm("Stop loading?")) {
startStopButton.setAttribute("disabled", true);
appendStatusHTML("<span class='redText'>Stopping after the current page...</span>\n");
interruptSignal = true;
}
}, false);
inserttoTable = pref.get("inserttoTable");
playsilentAudio = pref.get("playsilentAudio");
pageTimer = pref.get("pageTimer");
//Use crawl status to prevent the timer from going crazy.
let crawlStatus = "idle";
//Get the current page
getThisPage();
if (nextURL) {
if (playsilentAudio) {
//initialize audio context
if (!silentAudio) {
silentAudio = new SilentAudio();
}
silentAudio.play();
}
appendStatusHTML("Loading will take at least " + (pageTimer / 1000).toFixed(1) + " seconds per page.\n");
let crawlInterval = setInterval(function () {
if (crawlStatus == "idle") {
//Set crawl status to working
crawlStatus = "working";
pageCounter++;
//console.log("Current page: " + counter);
if (!nextURL) {
//All pages are done.
//Disable interval
clearInterval(crawlInterval);
setTimeout(function () {
finishCrawling();
}, 1000);
} else {
appendStatusHTML("Loading Page " + (pageCounter + 1) + "...\n");
let tryCount = 0;
let retryLimit = 10;
$.ajax({
url: nextURL,
type: 'GET',
success: function (sss) {
// console.time("parse page");
let parser = new DOMParser();
let docSSS = parser.parseFromString(sss, 'text/html');
//LATER cleanup this loop forEach
let trlen = docSSS.querySelectorAll(".itg .glname").length;
for (var j = 0; j < trlen; j++) {
let thislink;
//Extended mode
if (displayMode == "e") {
let row = docSSS.querySelectorAll(".itg > tbody > tr")[j];
thislink = row.querySelector(".glink").closest("a");
//get json data from row
jsonData.push(getJSONfromExtendedRow(row));
} else {
thislink = docSSS.querySelectorAll(".itg .glname a")[j];
}
getText(thislink);
}
//Bulk append the nodes
if (inserttoTable) {
if (displayMode != "t") {
//All except thumbnail mode
//Filter the table header
let rows = [...docSSS.querySelectorAll(".itg > tbody > tr")].filter((elem) => elem.querySelector(".glname"));
document.querySelector(".itg tbody").append(...rows);
} else {
document.querySelector(".itg").append(...docSSS.querySelectorAll(".itg > .gl1t"));
}
}
//interrupted with the stop button
if (interruptSignal) {
nextURL = "";
} else {
//Get nextURL
nextURL = docSSS.querySelector(".searchnav a[href*='next=']")?.href;
}
//Set crawl status to idle
crawlStatus = "idle";
// console.timeEnd("parse page");
},
error: function (xhr, textStatus, errorThrown) {
tryCount++;
if (tryCount <= retryLimit) {
//try again
$.ajax(this);
return;
}
//Must have failed all retries for this page
crawlStatus = "idle";
return;
/*
if (textStatus == 'timeout') {
}
if (xhr.status == 500) {
//handle error
} else {
//handle error
}
*/
}
});
}
}
}, 200 + pageTimer);
} else {
finishCrawling()
}
function finishCrawling() {
if (playsilentAudio && silentAudio) {
silentAudio.pause();
}
startStopButton.setAttribute("disabled", true);
startStopButton.setAttribute("style", "");
if (interruptSignal) {
appendStatusHTML("<span class='redText'>Loading Interrupted!</span>\n");
} else {
appendStatusHTML("Loading complete.\n");
}
appendStatusHTML("Copy above list and/or press Ctrl+S to save the page (use the \"complete\" or the \".mhtml\" option).\n");
appendStatusHTML("Info: <span class='blueText'>" + totalFavoritesCount + "</span> favorites total in all categories.\nConfirmation: <span class='blueText'>" + (document.querySelector("#exportFav").value.split("\n").length - 1) + "</span> favorites in above text area, <span class='blueText'>" + (document.querySelectorAll(".itg .glname a").length + document.querySelectorAll(".gl4e .glink").length) + "</span> favorites in below page.\n");
appendStatusHTML("If your gallery count doesn't add up, try again. You can load the pages faster now that you cached all the thumbnails.\n");
addRemoveButton(1);
if (displayMode == "e") {
appendStatusHTML("<a target='_blank' href='https://sadpanda-graphs.surge.sh/'>Example JSON use case (favorite tag charts)</a>\n");
document.querySelector("#SSEAFjsonData").textContent = JSON.stringify(jsonData);
}
crawlStatus = "idle";
}
}
/* --- JSON template info ---
Trying to stay compatible with the dnsev-h' JSON format.
Using gallery_info_full but obviously a ton of data is missing. Just compare the output.
Tags might be missing their alternative spellings ("hanafuda sakurano | hanafuda sakura" will only have "hanafuda sakurano")
My additions:
- "date_favorited": 1579748580000
- "user_rating": 4.5
- "favorites": { "note": "favorite note text" }
*/
class baseJSON {
constructor() {
this.data = {
"gallery_info_full": {
"gallery": {
"gid": null,
"token": null
},
"title": "",
// "title_original": "",
"date_uploaded": 0,
"date_favorited": 0,
"category": "",
"uploader": "",
"rating": {
"average": 0,
"count": 0
},
"user_rating": 0,
"favorites": {
"category": -1,
"category_title": "",
"note": "",
// "count": 0
},
// "parent": null,
// "newer_versions": [],
"thumbnail": "",
// "thumbnail_size": "large",
// "thumbnail_rows": 0,
"image_count": 0,
// "images_resized": false,
// "total_file_size_approx": 0,
// "visible": true,
// "visible_reason": "",
// "language": "Japanese",
// "translated": false,
"tags": {},
"tags_have_namespace": true,
// "torrent_count": 0,
// "archiver_key": "",
"source": "layout_extended",
"source_site": "exhentai",
"date_generated": 0
},
"source_script": "save-export-all-favorites",
};
}
}
//generated by countAllFavs()
let favoriteCategoryDict = {};
const categoryDict = {
"Doujinshi": "doujinshi",
"Manga": "manga",
"Artist CG": "artistcg",
"Game CG": "gamecg",
"Western": "western",
"Non-H": "non-h",
"Image Set": "imageset",
"Cosplay": "cosplay",
"Asian Porn": "asianporn",
"Misc": "misc"
};
//Parse extended layout rows for json data. This might break if another script adds something to the extended layout.
function getJSONfromExtendedRow(row) {
try {
let rowJson = new baseJSON().data;
let info = rowJson["gallery_info_full"];
let matchGID = row.querySelector("a").href.match(/\/g\/(\d+)\/(.*?)(\/)*$/);
info.gallery = {
"gid": matchGID[1],
"token": matchGID[2]
};
info.title = row.querySelector(".glink").textContent.trim();
info.date_uploaded = getTimestamp(row.querySelector("div[id*=posted_]").textContent);
info.date_favorited = getTimestamp(row.querySelector(".gldown").nextElementSibling.querySelectorAll("p")[1].textContent);
info.category = categoryDict[row.querySelector(".cn").textContent];
let uploaderNode = row.querySelector("a[href*='/uploader/']");
info.uploader = uploaderNode ? decodeURIComponent(uploaderNode.href.match(/uploader\/(.*)/)[1]) : null;
//User rating or Public rating. Can't get public rating if user rating exists.
let ratingNode = row.querySelector(".ir") || null;
if (ratingNode) {
let ratingMatch = ratingNode.attributes.style.value.match(/background-position:[- ]*(\d+?)px[- ]*(\d+?)px.*/);
if (ratingMatch.length >= 3) {
let ratingStars = 5 - (parseInt(ratingMatch[1]) / 16) - ((parseInt(ratingMatch[2]) == 21) ? 0.5 : 0);
if (ratingNode.classList.contains("irb") || ratingNode.classList.contains("irg") || ratingNode.classList.contains("irr")) {
//User rating
info.user_rating = ratingStars;
} else {
//Public rating
info.rating.average = ratingStars;
}
}
}
//Category (0 for the first category, 9 for the last category)
let categoryTitle = row.querySelector("div[id*=posted_]").title;
//Favorite note
let favNote = "";
let noteNode = row.querySelector(".glfnote");
if (noteNode.textContent != "") {
favNote = noteNode.textContent.match(/^Note: (.*)/)[1];
}
info.favorites = {
"category": favoriteCategoryDict[categoryTitle],
"category_title": categoryTitle,
"note": favNote
};
info.thumbnail = row.querySelector("img").src;
info.image_count = parseInt(row.querySelector(".gldown").previousElementSibling.textContent.match(/(\d+) pages/)[1]);
info.tags = generateTagListByNameSpace(row);
info.source_site = document.location.host.split(".")[0];
info.date_generated = Date.now();
return rowJson;
} catch (error) {
console.error("SSEAF: getJSONfromExtendedRow error!", error);
console.trace(this);
}
}
//generateTagListByNameSpace by Caca Ductile. Thank you.
function generateTagListByNameSpace(row) {
let tags = {};
row.querySelectorAll(".gt,.gtl,.gtw").forEach((tag) => {
const content = tag.title.split(":");
let namespace = content[0];
const value = content[1];
if (namespace == "" && value) {
//Handle: temp tags don't have a namespace in title ":temptag"
namespace = tag.closest(".glname tr").querySelector(".tc").textContent.split(":")[0] || "temp";
}
if (!tags[namespace]) {
tags[namespace] = [];
}
tags[namespace].push(value);
});
return tags;
}
//getTimestamp by dnsev-h (https://github.com/dnsev-h/x). Thank you.
function getTimestamp(text) {
let match = /([0-9]+)-([0-9]+)-([0-9]+)\s+([0-9]+):([0-9]+)/.exec(text);
if (match === null) {
return null;
}
return Date.UTC(parseInt(match[1], 10), // year
parseInt(match[2], 10) - 1, // month
parseInt(match[3], 10), // day
parseInt(match[4], 10), // hours
parseInt(match[5], 10), // minutes
0, // seconds
0); // milliseconds
}
function downloadFile(content, fileName, contentType) {
const a = document.createElement("a");
const file = new Blob([content], { type: contentType });
a.href = URL.createObjectURL(file);
a.download = fileName;
a.click();
}
function addMyStyle(styleID, styleCSS) {
var myStyle = document.createElement('style');
//myStyle.type = 'text/css';
myStyle.id = styleID;
myStyle.textContent = styleCSS;
document.querySelector("head").appendChild(myStyle);
}
function downloadJSON() {
//yyyy-mm-dd
let today = new Date();
today = new Date(today.getTime() - (today.getTimezoneOffset() * 60000))
today = today.toISOString().split('T')[0];
downloadFile(JSON.stringify(jsonData, null, 2), `Sadpanda-Favorites-${today}.json`, "text/plain");
}
