RateLimitedFetcher 网络请求层 — JM Shelf 推荐脚本的模块库，通过 @require 被主脚本引用。
Dit script moet niet direct worden geïnstalleerd - het is een bibliotheek voor andere scripts om op te nemen met de meta-richtlijn // @require https://update.sleazyfork.org/scripts/581103/1842602/JM%20Shelf%20-%20Network.js
// ==UserScript==
// @name JM Shelf - Network
// @namespace jmshelf-lib
// @version 1.0.0
// @author Kesdi
// @description RateLimitedFetcher 网络请求层 — JM Shelf 推荐脚本的模块库，通过 @require 被主脚本引用。
// @license MIT
// ==/UserScript==
//
// 此文件是 GreasyFork 库(library)，不直接安装。
// 请安装主脚本: JM Shelf 给杂鱼的个性化推荐
//
// ═══ [6] NETWORK (RateLimitedFetcher) ═══
// ============================================================
class RateLimitedFetcher {
constructor() {
this.queue = [];
this.running = false;
this.failCount = 0;
this.pausedUntil = 0;
}
async enqueue(url, parser = null, priority = 0) {
return new Promise((resolve, reject) => {
this.queue.push({ url, parser, priority, resolve, reject });
this.queue.sort((a, b) => b.priority - a.priority);
if (!this.running) this._process();
});
}
async _process() {
if (this.running) return;
this.running = true;
while (this.queue.length > 0) {
const now = Date.now();
if (now < this.pausedUntil) {
const waitMs = this.pausedUntil - now;
LOG.warn(`速率限制暂停中，等待 ${Math.round(waitMs / 1000)} 秒...`);
await sleep(waitMs);
this.pausedUntil = 0;
this.failCount = 0;
}
const item = this.queue.shift();
try {
const result = await this._fetch(item.url, item.parser);
item.resolve(result);
this.failCount = 0;
} catch (e) {
this.failCount++;
const itemFails = (item._failCount || 0) + 1;
item._failCount = itemFails;
if (itemFails >= 5) {
LOG.warn(`请求放弃 (${itemFails}次失败): ${item.url.substring(0,80)}`);
item.reject(e);
this.failCount = 0;
continue;
}
const backoff = Math.min(CONFIG.REQUEST_DELAY_MS * Math.pow(2, Math.min(itemFails, 6)), 900000);
// 超过10分钟 → 放弃并标记失败
item._totalWait = (item._totalWait || 0) + backoff;
if (item._totalWait >= 600000) {
LOG.error(`请求超时放弃 (${itemFails}次失败, 累计${Math.round(item._totalWait/60000)}分钟): ${item.url.substring(0,80)}`);
item.reject(new Error('TIMEOUT: backoff exceeded 10min'));
this.failCount = 0;
this._scanFailed = true;
continue;
}
LOG.error(`请求失败 (尝试${itemFails}): ${item.url.substring(0,80)} — ${Math.round(backoff/1000)}秒后重试 (不阻塞队列)`);
// 不阻塞: 推到队尾, 队列空时自动重新唤醒_process
setTimeout(() => { this.queue.push(item); this.queue.sort((a,b) => b.priority - a.priority); if (!this.running) this._process(); }, backoff);
}
if (this.queue.length > 0) {
await sleep(Math.max(CONFIG.REQUEST_DELAY_MS, 1000));
}
}
this.running = false;
}
_fetch(url, parser) {
return new Promise((resolve, reject) => {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 30000);
fetch(url, {
method: 'GET',
credentials: 'include',
signal: controller.signal,
})
.then(resp => {
clearTimeout(timeoutId);
if (resp.status === 429 || resp.status === 503 || resp.status === 403) {
reject(new Error(`HTTP ${resp.status}`));
return;
}
if (resp.status === 404) {
resolve(null);
return;
}
if (resp.status >= 500) {
reject(new Error(`HTTP ${resp.status}`));
return;
}
return resp.text();
})
.then(text => {
if (text === undefined) return;
try {
const result = parser ? parser(text, url) : text;
resolve(result);
} catch (e) {
reject(new Error('Parse error: ' + e.message));
}
})
.catch(e => {
clearTimeout(timeoutId);
if (e.name === 'AbortError') {
reject(new Error('Timeout'));
} else {
reject(new Error('Network error: ' + e.message));
}
});
});
}
get queueLength() { return this.queue.length; }
get isPaused() { return Date.now() < this.pausedUntil; }
}
const fetcher = new RateLimitedFetcher();