Download kemono post in a zip file
Από την
// ==UserScript==
// @name Kemono zip download
// @name:zh-CN Kemono 下载为ZIP文件
// @namespace https://greasyfork.org/users/667968-pyudng
// @version 0.4
// @description Download kemono post in a zip file
// @description:zh-CN 下载Kemono的内容为ZIP压缩文档
// @author PY-DNG
// @license MIT
// @match http*://*.kemono.su/*
// @match http*://*.kemono.party/*
// @require data:application/javascript,window.setImmediate%20%3D%20window.setImmediate%20%7C%7C%20((f%2C%20...args)%20%3D%3E%20window.setTimeout(()%20%3D%3E%20f(args)%2C%200))%3B
// @require https://update.greasyfork.org/scripts/456034/1532680/Basic%20Functions%20%28For%20userscripts%29.js
// @require https://cdnjs.cloudflare.com/ajax/libs/jszip/3.10.1/jszip.min.js
// @require https://update.greasyfork.org/scripts/458132/1138364/ItemSelector.js
// @connect kemono.su
// @connect kemono.party
// @icon https://kemono.su/favicon.ico
// @grant GM_xmlhttpRequest
// @grant GM_registerMenuCommand
// @run-at document-start
// ==/UserScript==
/* eslint-disable no-multi-spaces */
/* eslint-disable no-return-assign */
/* global LogLevel DoLog Err $ $All $CrE $AEL $$CrE addStyle detectDom destroyEvent copyProp copyProps parseArgs escJsStr replaceText getUrlArgv dl_browser dl_GM AsyncManager queueTask FunctionLoader loadFuncs require isLoaded */
/* global setImmediate, JSZip, ItemSelector */
(function __MAIN__() {
'use strict';
const CONST = {
TextAllLang: {
DEFAULT: 'en-US',
'zh-CN': {
DownloadZip: '下载为ZIP文件',
DownloadPost: '下载当前作品为ZIP文件',
DownloadCreator: '下载当前创作者为ZIP文件',
Downloading: '正在下载...',
SelectAll: '全选'
},
'en-US': {
DownloadZip: 'Download as ZIP file',
DownloadPost: 'Download post as ZIP file',
DownloadCreator: 'Download creator as ZIP file',
Downloading: 'Downloading...',
SelectAll: 'Select All'
}
}
};
// Init language
const i18n = Object.keys(CONST.TextAllLang).includes(navigator.language) ? navigator.language : CONST.TextAllLang.DEFAULT;
CONST.Text = CONST.TextAllLang[i18n];
loadFuncs([{
id: 'utils',
async func() {
const win = typeof unsafeWindow !== 'undefined' ? unsafeWindow : window;
function fillNumber(number, length) {
const str = number.toString();
return '0'.repeat(length - str.length) + str;
}
class ProgressManager {
/** @type {number} */
#steps;
/** @type {progressCallback} */
#callback;
/** @type {number} */
#finished;
/**
* This callback is called each time a promise resolves
* @callback progressCallback
* @param {number} resolved_count
* @param {number} total_count
*/
/**
* @param {number} steps - Total steps count of the task
*/
constructor(steps, callback) {
this.#steps = steps;
this.#callback = callback;
this.#finished = 0;
this.#callback(this.#finished, this.#steps);
}
async progress(promise) {
const val = await promise;
try {
this.#callback(++this.#finished, this.#steps);
} finally {
return val;
}
}
/**
* Resolves after all promise resolved, and callback each time one of them resolves
* @param {Array<Promise>} promises
* @param {progressCallback} callback
*/
static async all(promises, callback) {
const manager = new ProgressManager(promises.length, callback);
await Promise.all(promises.map(promise => manager.progress(promise, callback)));
}
}
const PerformanceManager = (function() {
class RunRecord {
static #id = 0;
/** @typedef {'initialized' | 'running' | 'finished'} run_status */
/** @type {number} */
id;
/** @type {number} */
start;
/** @type {number} */
end;
/** @type {number} */
duration;
/** @type {run_status} */
status;
/**
* Anything for programmers to mark and read, uses as a description for this run
* @type {*}
*/
info;
/**
* @param {*} [info] - Anything for programmers to mark and read, uses as a description for this run
*/
constructor(info) {
this.id = RunRecord.#id++;
this.status = 'initialized';
this.info = info;
}
run() {
const time = performance.now();
this.start = time;
this.status = 'running';
return this;
}
stop() {
const time = performance.now();
this.end = time;
this.duration = this.end - this.start;
this.status = 'finished';
return this;
}
}
class Task {
/** @typedef {number | string | symbol} task_id */
/** @type {task_id} */
id;
/** @type {RunRecord[]} */
runs;
/**
* @param {task_id} id
*/
constructor(id) {
this.id = id;
this.runs = [];
}
run(info) {
const record = new RunRecord(info);
record.run();
this.runs.push(record);
return record;
}
get time() {
return this.runs.reduce((time, record) => {
if (record.status === 'finished') {
time += record.duration;
}
return time;
}, 0)
}
}
class PerformanceManager {
/** @type {Task[]} */
tasks;
constructor() {
this.tasks = [];
}
/**
* @param {task_id} id
* @returns {Task | null}
*/
getTask(id) {
return this.tasks.find(task => task.id === id);
}
/**
* Creates a new task
* @param {task_id} id
* @returns {Task}
*/
newTask(id) {
Assert(!this.getTask(id), `given task id ${escJsStr(id)} is already in use`, TypeError);
const task = new Task(id);
this.tasks.push(task);
return task;
}
/**
* Runs a task
* @param {task_id} id
* @param {*} run_info - Anything for programmers to mark and read, uses as a description for this run
* @returns {RunRecord}
*/
run(task_id, run_info) {
const task = this.getTask(task_id);
Assert(task, `task of id ${escJsStr(task_id)} not found`, TypeError);
return task.run(run_info);
}
totalTime(id) {
if (id) {
return this.getTask(id).time;
} else {
return this.tasks.reduce((timetable, task) => {
timetable[task.id] = task.time;
return timetable;
}, {});
}
}
meanTime(id) {
if (id) {
const task = this.getTask(id);
return task.time / task.runs.length;
} else {
return this.tasks.reduce((timetable, task) => {
timetable[task.id] = task.time / task.runs.length;
return timetable;
}, {});
}
}
}
return PerformanceManager;
}) ();
return {
window: win,
fillNumber, ProgressManager, PerformanceManager
}
}
}, {
id: 'api',
desc: 'api for kemono',
async func() {
let api_key = null;
const Posts = {
/**
* Get a list of creator posts
* @param {string} service - The service where the post is located
* @param {number | string} creator_id - The ID of the creator
* @param {string} [q] - Search query
* @param {number} [o] - Result offset, stepping of 50 is enforced
*/
posts(service, creator_id, q, o) {
const search = {};
q && (search.q = q);
o && (search.o = o);
return callApi({
endpoint: `/${service}/user/${creator_id}`,
search
});
},
/**
* Get a specific post
* @param {string} service
* @param {number | string} creator_id
* @param {number | string} post_id
* @returns {Promise<Object>}
*/
post(service, creator_id, post_id) {
return callApi({
endpoint: `/${service}/user/${creator_id}/post/${post_id}`
});
}
};
const Creators = {
/**
* Get a creator
* @param {string} service - The service where the creator is located
* @param {number | string} creator_id - The ID of the creator
* @returns
*/
profile(service, creator_id) {
return callApi({
endpoint: `/${service}/user/${creator_id}/profile`
});
}
};
const Custom = {
/**
* Get a list of creator's ALL posts, calling Post.posts for multiple times and joins the results
* @param {string} service - The service where the post is located
* @param {number | string} creator_id - The ID of the creator
* @param {string} [q] - Search query
*/
async all_posts(service, creator_id, q) {
const posts = [];
let offset = 0;
let api_result = null;
while (!api_result || api_result.length === 50) {
api_result = await Posts.posts(service, creator_id, q, offset);
posts.push(...api_result);
offset += 50;
}
return posts;
}
};
const API = {
get key() { return api_key; },
set key(val) { api_key = val; },
Posts, Creators,
Custom,
callApi
};
return API;
/**
* callApi detail object
* @typedef {Object} api_detail
* @property {string} endpoint - api endpoint
* @property {Object} [search] - search params
* @property {string} [method='GET']
* @property {boolean | string} [auth=false] - whether to use api-key in this request; true for send, false for not, and string for sending a specific key (instead of pre-configured `api_key`); defaults to false
*/
/**
* Do basic kemono api request
* This is the queued version of _callApi
* @param {api_detail} detail
* @returns
*/
function callApi(...args) {
return queueTask(() => _callApi(...args), 'callApi');
}
/**
* Do basic kemono api request
* @param {api_detail} detail
* @returns
*/
function _callApi(detail) {
const search_string = new URLSearchParams(detail.search).toString();
const url = `https://kemono.su/api/v1/${detail.endpoint.replace(/^\//, '')}` + (search_string ? '?' + search_string : '');
const method = detail.method ?? 'GET';
const auth = detail.auth ?? false;
return new Promise((resolve, reject) => {
auth && api_key === null && reject('api key not found');
const options = {
method, url,
headers: {
accept: 'application/json'
},
onload(e) {
try {
e.status === 200 ? resolve(JSON.parse(e.responseText)) : reject(e.responseText);
} catch(err) {
reject(err);
}
},
onerror: err => reject(err)
}
if (typeof auth === 'string') {
options.headers.Cookie = auth;
} else if (auth === true) {
options.headers.Cookie = api_key;
}
GM_xmlhttpRequest(options);
});
}
}
}, {
id: 'downloader',
desc: 'core zip download utils',
dependencies: ['utils', 'api'],
async func() {
const utils = require('utils');
const API = require('api');
// Performance record
const perfmon = new utils.PerformanceManager();
perfmon.newTask('fetchPost');
perfmon.newTask('saveAs');
perfmon.newTask('_fetchBlob');
class DownloaderItem {
/** @typedef {'file' | 'folder'} downloader_item_type */
/**
* Name of the item, CANNOT BE PATH
* @type {string}
*/
name;
/** @type {downloader_item_type} */
type;
/**
* @param {string} name
* @param {downloader_item_type} type
*/
constructor(name, type) {
this.name = name;
this.type = type;
}
}
class DownloaderFile extends DownloaderItem{
/** @type {Blob} */
data;
/** @type {Date} */
date;
/** @type {string} */
comment;
/**
*
* @param {string} name - name only, CANNOT BE PATH
* @param {Blob} data
* @param {Object} detail
* @property {Date} [date]
* @property {string} [comment]
*/
constructor(name, data, detail) {
super(name, 'file');
this.data = data;
Object.assign(this, detail);
}
zip(jszip_instance) {
const z = jszip_instance ?? new JSZip();
const options = {};
this.date && (options.date = this.date);
this.comment && (options.comment = this.comment);
z.file(this.name, this.data, options);
return z;
}
}
class DownloaderFolder extends DownloaderItem {
/** @type {Array<DownloaderFile | DownloaderFolder>} */
children;
/**
* @param {string} name - name only, CANNOT BE PATH
* @param {Array<DownloaderFile | DownloaderFolder>} [children]
*/
constructor(name, children) {
super(name, 'folder');
this.children = children && children.length ? children : [];
}
zip(jszip_instance) {
const z = jszip_instance ?? new JSZip();
for (const child of this.children) {
switch (child.type) {
case 'file': {
child.zip(z);
break;
}
case 'folder': {
const sub_z = z.folder(child.name);
child.zip(sub_z);
}
}
}
return z;
}
}
/**
* @typedef {Object} JSZip
*/
/**
* @typedef {Object} zipObject
*/
/**
* Download one post in zip file
* @param {string} service
* @param {number | string} creator_id
* @param {number | string} post_id
* @param {function} [callback] - called each time made some progress, with two numeric arguments: finished_steps and total_steps
*/
async function downloadPost(service, creator_id, post_id, callback = function() {}) {
const manager = new utils.ProgressManager(3, callback);
const data = await manager.progress(API.Posts.post(service, creator_id, post_id));
const folder = await manager.progress(fetchPost(data));
const zip = folder.zip();
const filename = `${data.post.title}.zip`;
manager.progress(await saveAs(filename, zip));
}
/**
* @typedef {Object} PostInfo
* @property {string} service,
* @property {number} creator_id
* @property {number} post_id
*/
/**
* Download one or multiple posts in zip file, one folder for each post
* @param {PostInfo | PostInfo[]} posts
* @param {*} callback - Progress callback, like downloadPost's callback param
*/
async function downloadPosts(posts, filename, callback = function() {}) {
Array.isArray(posts) || (posts = [posts]);
const manager = new utils.ProgressManager(2, callback);
// Fetch posts
const post_folders = await manager.progress(Promise.all(
posts.map(async post => {
const data = await API.Posts.post(post.service, post.creator_id, post.post_id);
const folder = await fetchPost(data);
return folder;
})
));
// Merge all post's folders into one
const folder = new DownloaderFolder(filename);
post_folders.map(async post_folder => {
folder.children.push(post_folder);
})
// Convert folder to zip
const zip = folder.zip();
// Deliver to user
await manager.progress(saveAs(filename, zip));
}
/**
* Fetch one post
* @param {Object} api_data - kemono post api returned data object
* @returns {Promise<DownloaderFolder>}
*/
async function fetchPost(api_data) {
const perfmon_run = perfmon.run('fetchPost', api_data);
const date = new Date(api_data.post.edited);
const folder = new DownloaderFolder(api_data.post.title, [
new DownloaderFile('data.json', JSON.stringify(api_data)),
new DownloaderFile('content.html', api_data.post.content, { date })
]);
const tasks = [
// api_data.post.file
(async function(file) {
if (!file.path) { return; }
const name = `file-${getFileName(file)}`;
const url = 'https://n1.kemono.su/data' + file.path;
const blob = await fetchBlob(url);
folder.children.push(new DownloaderFile(
name, blob, {
date,
comment: JSON.stringify(file)
}
));
}) (api_data.post.file),
// api_data.post.attachments
...api_data.post.attachments.map(async (attachment, i) => {
const prefix = utils.fillNumber(i+1, api_data.post.attachments.length.toString().length);
const name = `${prefix}-${getFileName(attachment)}`;
const url = 'https://n1.kemono.su/data' + attachment.path;
const blob = await fetchBlob(url);
const attachments_folder = new DownloaderFolder('attachments');
attachments_folder.children.push(new DownloaderFile(
name, blob, {
date,
comment: JSON.stringify(attachment)
}
));
folder.children.push(attachments_folder);
}),
];
await Promise.all(tasks);
perfmon_run.stop();
return folder;
function getFileName(file) {
return file.name || file.path.slice(file.path.lastIndexOf('/')+1);
}
}
/**
* Deliver zip file to user
* @param {string} filename - filename (with extension, e.g. "file.zip")
* @param {JSZip} zip,
* @param {string | null} comment - zip file comment
*/
async function saveAs(filename, zip, comment=null) {
const perfmon_run = perfmon.run('saveAs', filename);
const options = { type: 'blob' };
comment !== null && (options.comment = comment);
const blob = await zip.generateAsync(options);
const url = URL.createObjectURL(blob);
const a = $$CrE({
tagName: 'a',
attrs: {
download: filename,
href: url
}
});
a.click();
perfmon_run.stop();
}
/**
* Fetch blob data from given url \
* Queued function of _fetchBlob
* @param {string} url
* @returns {Promise<Blob>}
*/
function fetchBlob(...args) {
if (!fetchBlob.initialized) {
queueTask.fetchBlob = {
max: 3,
sleep: 0
};
fetchBlob.initialized = true;
}
return queueTask(() => _fetchBlob(...args), 'fetchBlob');
}
/**
* Fetch blob data from given url
* @param {string} url
* @param {number} [retry=3] - times to retry before throwing an error
* @returns {Promise<Blob>}
*/
async function _fetchBlob(url, retry = 3) {
const perfmon_run = perfmon.run('_fetchBlob', url);
const blob = await new Promise((resolve, reject) => {
GM_xmlhttpRequest({
method: 'GET', url,
responseType: 'blob',
onload(e) {
e.status === 200 ? resolve(e.response) : onerror(e)
},
onerror
});
async function onerror(err) {
if (retry) {
const result = await _fetchBlob(url, retry - 1);
resolve(result);
} else {
reject(err)
}
}
});
perfmon_run.stop();
return blob;
}
return {
downloadPost, downloadPosts,
fetchPost, saveAs,
perfmon
};
}
}, {
id: 'user-interface',
dependencies: ['utils', 'api', 'downloader'],
async func() {
const utils = require('utils');
const api = require('api');
const downloader = require('downloader');
let downloading = false;
let selector = null;
// Console User Interface
const ConsoleUI = utils.window.ZIP = {
version: GM_info.script.version,
require,
api, downloader,
get ui() { return require('user-interface') }
};
// Menu User Interface
GM_registerMenuCommand(CONST.Text.DownloadPost, downloadCurrentPost);
GM_registerMenuCommand(CONST.Text.DownloadCreator, downloadCurrentCreator);
// Graphical User Interface
// Make button
const dlbtn = $$CrE({
tagName: 'button',
styles: {
'background-color': 'transparent',
'color': 'white',
'border': 'transparent'
},
listeners: [['click', userDownload]]
});
const dltext = $$CrE({
tagName: 'span',
props: {
innerText: CONST.Text.DownloadZip
}
});
const dlprogress = $$CrE({
tagName: 'span',
styles: {
display: 'none',
'margin-left': '10px'
}
});
dlbtn.append(dltext);
dlbtn.append(dlprogress);
// Place button each time a new action panel appears (meaning navigating into a post page)
let observer;
detectDom({
selector: '.post__actions, .user-header__actions',
callback: action_panel => {
// Hide dlprogress, its content is still for previous page
dlprogress.style.display = 'none';
// Append to action panel
action_panel.append(dlbtn);
// Disconnect old observer
observer?.disconnect();
// Observe action panel content change, always put download button in last place
observer = detectDom({
root: action_panel,
selector: 'button',
callback: btn => btn !== dlbtn && action_panel.append(dlbtn)
});
}
});
return {
ConsoleUI,
dlbtn, dltext,
get ['selector']() { return selector; },
downloadCurrentPost, downloadCurrentCreator,
getCurrentPost, getCurrentCreator
}
function userDownload() {
const page_type = getPageType();
const func = ({
post: downloadCurrentPost,
creator: downloadCurrentCreator
})[page_type] ?? function() {};
return func();
}
async function downloadCurrentPost() {
const post_info = getCurrentPost();
if (downloading) { return; }
if (!post_info) { return; }
try {
downloading = true;
dlprogress.style.display = 'inline';
dltext.innerText = CONST.Text.Downloading;
await downloader.downloadPost(
post_info.service,
post_info.creator_id,
post_info.post_id,
on_progress
);
} finally {
downloading = false;
dltext.innerText = CONST.Text.DownloadZip;
}
function on_progress(finished_steps, total_steps) {
dlprogress.innerText = `(${finished_steps}/${total_steps})`;
}
}
async function downloadCurrentCreator() {
const creator_info = getCurrentCreator();
if (downloading) { return; }
if (!creator_info) { return; }
try {
downloading = true;
const profile = await api.Creators.profile(creator_info.service, creator_info.creator_id);
const posts = await api.Custom.all_posts(creator_info.service, creator_info.creator_id);
const selected_posts = await new Promise((resolve, reject) => {
if (!selector) {
selector = new ItemSelector();
selector.setTheme('dark');
selector.elements.container.style.setProperty('z-index', '1');
}
selector.show({
text: CONST.Text.SelectAll,
children: posts.map(post => ({
post,
text: post.title
}))
}, {
title: CONST.Text.DownloadCreator,
onok(e, json) {
const posts = json.children.map(obj => obj.post);
resolve(posts);
},
oncancel: e => resolve(null),
onclose: e => resolve(null)
});
});
if (selected_posts) {
dlprogress.style.display = 'inline';
dltext.innerText = CONST.Text.Downloading;
await downloader.downloadPosts(selected_posts.map(post => ({
service: creator_info.service,
creator_id: creator_info.creator_id,
post_id: post.id
})), `${profile.name}.zip`, on_progress);
}
} finally {
downloading = false;
dltext.innerText = CONST.Text.DownloadZip;
}
function on_progress(finished_steps, total_steps) {
dlprogress.innerText = `(${finished_steps}/${total_steps})`;
}
}
/**
* @typedef {Object} PostInfo
* @property {string} service,
* @property {number} creator_id
* @property {number} post_id
*/
/**
* Get post info in current page
* @returns {PostInfo | null}
*/
function getCurrentPost() {
const regpath = /^\/([a-zA-Z]+)\/user\/(\d+)\/post\/(\d+)/;
const match = location.pathname.match(regpath);
if (!match) {
return null;
} else {
return {
service: match[1],
creator_id: parseInt(match[2], 10),
post_id: parseInt(match[3], 10)
}
}
}
/**
* @typedef {Object} CreatorInfo
* @property {string} service
* @property {number} creator_id
*/
/**
* Get creator info in current page
* @returns {CreatorInfo | null}
*/
function getCurrentCreator() {
const regpath = /^\/([a-zA-Z]+)\/user\/(\d+)/;
const match = location.pathname.match(regpath);
if (!match) {
return null;
} else {
return {
service: match[1],
creator_id: parseInt(match[2], 10)
}
}
}
/** @typedef { 'post' | 'creator' } page_type */
/**
* @returns {page_type}
*/
function getPageType() {
const matchers = {
post: {
type: 'regpath',
value: /^\/([a-zA-Z]+)\/user\/(\d+)\/post\/(\d+)/
},
creator: {
type: 'func',
value: () => /^\/([a-zA-Z]+)\/user\/(\d+)/.test(location.pathname) && !location.pathname.includes('/post/')
},
}
for (const [type, matcher] of Object.entries(matchers)) {
if (FunctionLoader.testCheckers(matcher)) {
return type;
}
}
}
},
}]);
}) ();