Hides the images, videos and GIFs you blacklist on 4chan threads, board indexes and the catalog: the exact file (MD5), the same picture resized or recompressed, and edited copies; videos and GIFs by their thumbnail. Manual references only, reversible stubs, categories with their own settings. Vanilla 4chan, 4chan X, XT, XZ, neXT, StyleChan and OneeChan.
// ==UserScript==
// @name 4chan Better Filter
// @namespace https://greasyfork.org/users/1659081
// @version 1.3.0
// @description Hides the images, videos and GIFs you blacklist on 4chan threads, board indexes and the catalog: the exact file (MD5), the same picture resized or recompressed, and edited copies; videos and GIFs by their thumbnail. Manual references only, reversible stubs, categories with their own settings. Vanilla 4chan, 4chan X, XT, XZ, neXT, StyleChan and OneeChan.
// @match https://boards.4chan.org/*
// @grant GM_getValue
// @grant GM_setValue
// @grant GM_addStyle
// @grant GM_xmlhttpRequest
// @grant GM.getValue
// @grant GM.setValue
// @grant GM.listValues
// @grant GM.xmlHttpRequest
// @connect i.4cdn.org
// @connect 4chenz.github.io
// @connect archive.4plebs.org
// @connect i.4pcdn.org
// @connect desuarchive.org
// @connect desu-usergeneratedcontent.xyz
// @connect boards.fireden.net
// @connect img-lb.fireden.net
// @connect arch.b4k.dev
// @connect arch-img.b4k.dev
// @connect archived.moe
// @connect thebarchive.com
// @connect archiveofsins.com
// @connect archive.palanq.win
// @connect archive-media.palanq.win
// @run-at document-idle
// @noframes
// @author xxxmikaluvr2000xxx
// @license MIT
// ==/UserScript==
(async () => {
'use strict';
const VERSION = '1.3.0';
// Violentmonkey runs a script that uses @grant inside "with (wrapper) with (context)", the wrapper a
// Proxy of the page's window, so every global name (Math, Float64Array, Infinity...) is looked up
// through it, each time. In the hashing and matching loops that makes edited-copy matching much
// slower. Local names for the built-ins they use are found directly.
const { Math, Infinity, NaN, Number, String, Array, Object, Map, Set, WeakMap, JSON, parseInt,
Float64Array, Float32Array, Uint8Array, Uint8ClampedArray, Uint32Array, Int32Array } = globalThis;
// The matching code, as one function: the page takes its names below, and background workers run a copy
// of the same source (see startWorkers()).
function mbgxCore() {
// #region MBGX-CORE
// Pure hashing/matching/link functions, kept free of the DOM and userscript APIs so that they can
// be tested on their own.
const HASH_ALGO = 'box-v1';
/** RGBA (alpha composited over white) -> luma with the Rec.601 weights PIL's convert('L') uses. */
function toGray(rgba, w, h) {
const gray = new Float64Array(w * h);
for (let i = 0, p = 0; p < gray.length; i += 4, p++) {
const alpha = rgba[i + 3] / 255, bg = 255 * (1 - alpha);
gray[p] = (rgba[i] * alpha + bg) * 0.299 + (rgba[i + 1] * alpha + bg) * 0.587 + (rgba[i + 2] * alpha + bg) * 0.114;
}
return gray;
}
/** For each destination cell: flat [srcIndex, coveredFraction, ...] pairs. */
function axisWeights(src, dst) {
const scale = src / dst, out = [];
for (let o = 0; o < dst; o++) {
const start = o * scale, end = start + scale, ws = [];
for (let i = Math.floor(start); i < Math.min(src, Math.ceil(end)); i++) {
const w = Math.min(end, i + 1) - Math.max(start, i);
if (w > 1e-9) ws.push(i, w);
}
out.push(ws);
}
return out;
}
/** Exact area-average (box filter) resize. Deterministic, unlike canvas drawImage() downscaling. */
function boxResize(gray, w, h, tw, th) {
const wx = axisWeights(w, tw), wy = axisWeights(h, th);
const tmp = new Float64Array(tw * h), out = new Float64Array(tw * th);
for (let y = 0; y < h; y++) for (let x = 0; x < tw; x++) {
const ws = wx[x]; let s = 0;
for (let k = 0; k < ws.length; k += 2) s += gray[y * w + ws[k]] * ws[k + 1];
tmp[y * tw + x] = s;
}
const norm = (w / tw) * (h / th);
for (let y = 0; y < th; y++) for (let x = 0; x < tw; x++) {
const ws = wy[y]; let s = 0;
for (let k = 0; k < ws.length; k += 2) s += tmp[ws[k] * tw + x] * ws[k + 1];
out[y * tw + x] = s / norm;
}
return out;
}
function bitsToHex(bits) {
let s = '';
for (let i = 0; i < bits.length; i += 4) s += ((bits[i] << 3) | (bits[i + 1] << 2) | (bits[i + 2] << 1) | bits[i + 3]).toString(16);
return s;
}
/**
* 64-bit aHash (8x8 cells, cell > mean) and 64-bit dHash (9x8 cells, right > left; the
* Python imagehash convention). Hashes are 16-char hex strings, never JS Numbers.
* `low` marks near-uniform images that would match too many unrelated images.
*/
function hashPixels(rgba, w, h) {
return hashGray(toGray(rgba, w, h), w, h);
}
function hashGray(gray, w, h) {
const n = w * h;
let mean = 0;
for (let i = 0; i < n; i++) mean += gray[i];
mean /= n;
let variance = 0;
for (let i = 0; i < n; i++) variance += (gray[i] - mean) ** 2;
const std = Math.sqrt(variance / n);
const g8 = boxResize(gray, w, h, 8, 8), g9 = boxResize(gray, w, h, 9, 8);
const avg = g8.reduce((s, v) => s + v, 0) / 64;
const aBits = [], dBits = [];
for (let i = 0; i < 64; i++) aBits.push(g8[i] > avg ? 1 : 0);
for (let y = 0; y < 8; y++) for (let x = 0; x < 8; x++) dBits.push(g9[y * 9 + x + 1] > g9[y * 9 + x] ? 1 : 0);
const ones = aBits.reduce((s, b) => s + b, 0);
const { steps, resid } = cellInfo(g8, g9);
return { algo: HASH_ALGO, a: bitsToHex(aBits), d: bitsToHex(dBits), w, h,
std: Math.round(std * 100) / 100, steps, resid: Math.round(resid * 10) / 10,
low: ones < 5 || ones > 59 || std < 2 || steps < LOW_STEPS || resid < LOW_RESID };
}
/**
* How much of a thumbnail's hashes is picture rather than noise:
* steps - dHash comparisons whose two cells differ by more than 4 grey levels (of 64). Black and
* white pictures, title cards and text screenshots have few: their other bits are noise.
* resid - grey levels left in the 8x8 cells after the best-fitting plane (a straight ramp). Fades
* and gradients are almost all ramp, so unrelated ones share both hashes.
* Thumbnails below either limit are near-blank: filtered by picture only by references made from a
* near-blank thumbnail on purpose (marked `blank`); their exact file always is.
*/
const LOW_STEPS = 12, LOW_RESID = 5;
/**
* How well a video's or GIF's thumbnail (its first frame) can stand for it: 'blank' (see
* `low` above), 'weak' (fewer than WEAK.steps clear steps: mostly one flat colour, so other videos
* that open the same way can match it; the blacklist window warns), or null.
*/
const WEAK = { steps: 24 };
function thumbnailWeakness(fp) {
if (!fp) return null;
return fp.low ? 'blank' : fp.steps < WEAK.steps ? 'weak' : null;
}
function cellInfo(g8, g9) {
let steps = 0;
for (let y = 0; y < 8; y++) for (let x = 0; x < 8; x++) if (Math.abs(g9[y * 9 + x + 1] - g9[y * 9 + x]) > 4) steps++;
// least-squares plane v = c + bx*x + by*y over the 8x8 cells (x and y are centred, so the fit separates)
let sv = 0, sxv = 0, syv = 0;
for (let y = 0; y < 8; y++) for (let x = 0; x < 8; x++) { const v = g8[y * 8 + x]; sv += v; sxv += (x - 3.5) * v; syv += (y - 3.5) * v; }
const c = sv / 64, bx = sxv / 336, by = syv / 336; // sum of (x - 3.5)^2 over the 64 cells = 336
let rr = 0;
for (let y = 0; y < 8; y++) for (let x = 0; x < 8; x++) rr += (g8[y * 8 + x] - (c + bx * (x - 3.5) + by * (y - 3.5))) ** 2;
return { steps, resid: Math.sqrt(rr / 64) };
}
const HEX64 = /^[0-9a-f]{16}$/;
/** Number of set bits in a 32-bit integer. */
function popcount32(v) {
v -= (v >>> 1) & 0x55555555;
v = (v & 0x33333333) + ((v >>> 2) & 0x33333333);
return (Math.imul((v + (v >>> 4)) & 0x0f0f0f0f, 0x01010101) >>> 24);
}
/** Hamming distance between two 16-digit hex hashes; 999 if either is malformed. */
function hamming(x, y) {
if (!HEX64.test(x) || !HEX64.test(y)) return 999;
return popcount32(parseInt(x.slice(0, 8), 16) ^ parseInt(y.slice(0, 8), 16)) +
popcount32(parseInt(x.slice(8), 16) ^ parseInt(y.slice(8), 16));
}
/**
* Reference database prepared for matching, rebuilt in memory whenever references, categories or
* settings change (never stored). Usable references of one hash method are kept in their original
* order, their hashes as four 32-bit words each (aHash high/low, dHash high/low), plus:
* - per board, the references that apply there (built on first use);
* - per reference, which matching rules it uses: 0 = the general settings passed to
* matchPrepared(), others = a category's own rules.
* Categories (cats: [{id, enabled, boards, rules}]) are resolved here, not in the matching loop:
* references in a disabled category are left out; a reference applies to its own boards if it has
* any, else its category's, else all boards. A new hash method later means another prepared table.
*/
function prepareReferences(seeds, algo = HASH_ALGO, cats = [], opts = {}) {
// Which checks each entry is in: a category can switch exact files, edited copies, heavily cropped
// copies (pieces) and close variants (its settings);
// otherwise the general settings in opts (md5, edited, pieces, close; left out: on, as the tools expect).
const on = (own, k) => (own && typeof own[k] === 'boolean' ? own[k] : opts[k] !== false);
const extFlags = [], closeFlags = [];
const catById = new Map(cats.map(c => [c.id, c]));
const ownOf = new Map(cats.map(c => [c.id, c.settings || null]));
const list = [], scopes = [], ruleIdx = [], rules = [null], ruleOfCat = new Map();
for (const s of seeds) {
if (!s || s.algo !== algo || !HEX64.test(s.a) || !HEX64.test(s.d)) continue;
const c = catById.get(s.cat);
if (c && c.enabled === false) continue;
let r = 0;
if (c && c.rules) {
if (!ruleOfCat.has(c.id)) { ruleOfCat.set(c.id, rules.length); rules.push(c.rules); }
r = ruleOfCat.get(c.id);
}
const scope = Array.isArray(s.boards) && s.boards.length ? s.boards : c && Array.isArray(c.boards) && c.boards.length ? c.boards : null;
const rule = c && ownOf.get(c.id), edited = on(rule, 'edited'), close = on(rule, 'close');
list.push(s);
ruleIdx.push(r);
scopes.push(scope);
extFlags.push(edited);
closeFlags.push(close);
// Its main picture (a collage's largest panel, a picture without its added band) as a second entry
// of the same reference: matches report the reference itself, marked "part".
const part = s.part;
if (part && HEX64.test(part.a) && HEX64.test(part.d)) {
list.push(Object.assign(Object.create(s), { a: part.a, d: part.d, w: part.w, h: part.h, detail: part.detail, isPart: true }));
ruleIdx.push(r);
scopes.push(scope);
extFlags.push(edited);
closeFlags.push(close);
}
// Its pieces (piecesOf()) likewise: heavily cropped copies are copies of one of them.
// Only where heavily cropped copies are on (they are edited copies, so those must be on too).
const pieces = edited && on(rule, 'pieces');
const pcs = !pieces || typeof s.pieces !== 'string' ? [] : unpackPieces(s.pieces);
const PW = Math.max(1, ...pcs.map(p => p ? p.w : 0)), PH = Math.max(1, ...pcs.map(p => p ? p.h : 0));
for (const pc of pcs) {
if (!pc || !HEX64.test(pc.a) || !HEX64.test(pc.d)) continue;
// which piece it is, by its size against the largest (pieces nearly blank are not stored, so not by order)
const rw = pc.w / PW, rh = pc.h / PH, pieceKind = rw < 0.65 && rh < 0.65 ? 'centre' : Math.min(rw, rh) < 0.6 ? 'half' : 'strip';
list.push(Object.assign(Object.create(s), { a: pc.a, d: pc.d, w: pc.w, h: pc.h, detail: pc.detail, isPart: true, isPiece: true, pieceKind }));
ruleIdx.push(r);
scopes.push(scope);
extFlags.push(true);
closeFlags.push(false); // a piece is not "a very similar picture"
}
}
const words = new Uint32Array(list.length * 4), unrestricted = [], restricted = new Map();
list.forEach((s, i) => {
words[i * 4] = parseInt(s.a.slice(0, 8), 16);
words[i * 4 + 1] = parseInt(s.a.slice(8), 16);
words[i * 4 + 2] = parseInt(s.d.slice(0, 8), 16);
words[i * 4 + 3] = parseInt(s.d.slice(8), 16);
if (!scopes[i]) unrestricted.push(i);
else for (const b of new Set(scopes[i])) restricted.has(b) ? restricted.get(b).push(i) : restricted.set(b, [i]);
});
const n = rules.length, ruleA = new Int32Array(n), ruleD = new Int32Array(n), ruleM = new Int32Array(n), ruleE = new Uint8Array(n);
rules.forEach((r, i) => { if (r) { ruleA[i] = r.a; ruleD[i] = r.d; ruleM[i] = r.margin; ruleE[i] = r.mode === 'either' ? 1 : 0; } });
// Exact files: every reference with an MD5 (picture references that kept theirs, and MD5-only
// ones), with the same category and board rules. MD5 -> indices into md5Seeds.
const md5 = new Map(), md5Seeds = [], md5Scopes = [];
for (const s of seeds) {
const c = s && catById.get(s.cat);
if (!s || (c && c.enabled === false) || !on(c && ownOf.get(c.id), 'md5')) continue;
const scope = Array.isArray(s.boards) && s.boards.length ? s.boards : c && Array.isArray(c.boards) && c.boards.length ? c.boards : null;
const k = normMd5(s.md5);
if (!k) continue;
if (md5.has(k)) md5.get(k).push(md5Seeds.length); else md5.set(k, [md5Seeds.length]);
md5Seeds.push(s);
md5Scopes.push(scope);
}
const extOK = Uint8Array.from(extFlags, Number), closeOK = Uint8Array.from(closeFlags, Number);
return { algo, seeds: list, words, details: new Map(), unrestricted, restricted, perBoard: new Map(), ruleIdx: Int32Array.from(ruleIdx), rules, ruleA, ruleD, ruleM, ruleE,
extOK, closeOK, anyExt: extOK.includes(1), anyClose: closeOK.includes(1),
md5, md5Seeds, md5Scopes };
}
/**
* 4chan's MD5 of a file as it appears in data-md5 and in 4chan X filter rules: base64 of the 16
* bytes, "==" included. Also accepts URL-safe base64 (archives), missing padding and 32-digit hex.
* Returns null for anything else.
*/
function normMd5(v) {
if (typeof v !== 'string') return null;
const s = v.trim();
if (/^[0-9a-f]{32}$/i.test(s)) {
let bytes = '';
for (let i = 0; i < 32; i += 2) bytes += String.fromCharCode(parseInt(s.slice(i, i + 2), 16));
return btoa(bytes);
}
// 16 bytes = 22 base64 digits, the last carrying only 2 bits (so it is A, Q, g or w).
const m = /^([A-Za-z0-9+/_-]{21}[AQgw])(?:==)?$/.exec(s);
return m ? m[1].replace(/-/g, '+').replace(/_/g, '/') + '==' : null;
}
/** The reference whose MD5 is this file's and that applies on `board`, unless whitelisted; else null. */
function matchMd5(md5, db, allowSet, board) {
const k = normMd5(md5);
if (!k || allowSet.has(`md5 ${k}`)) return null;
for (const i of db.md5.get(k) || []) {
const scope = db.md5Scopes[i], s = db.md5Seeds[i];
// A picture reference's own hashes on the whitelist: this exact file was marked a false positive.
if ((!scope || scope.includes(board)) && !allowSet.has(`${s.a} ${s.d}`)) return s;
}
return null;
}
/** Indices of the references that apply on `board`, in original order (cached per board). */
function boardIndex(db, board) {
let idx = db.perBoard.get(board);
if (!idx) {
const extra = db.restricted.get(board);
idx = Int32Array.from(extra ? db.unrestricted.concat(extra).sort((x, y) => x - y) : db.unrestricted);
db.perBoard.set(board, idx);
}
return idx;
}
/**
* Whitelist as a set of "aHash dHash" keys and "md5 <MD5>" keys (an entry can have either or both);
* entries without a string dHash never match.
*/
function prepareAllow(allow) {
const keys = new Set();
for (const x of allow) {
if (!x) continue;
if (typeof x.a === 'string' && typeof x.d === 'string') keys.add(`${x.a} ${x.d}`);
const k = normMd5(x.md5);
if (k) keys.add(`md5 ${k}`);
}
return keys;
}
/**
* Compare a fingerprint with the prepared references that apply on `board`.
* cfg: the general rules {a, d, mode: 'both'|'either', margin}, used by references without
* category rules. Returns {match: {seed, da, dd, rule} | null, best: closest reference or null,
* near, allowed, low}; `rule` is the rules that reference is judged by.
* Every applicable reference is scanned (no early exit): `best` and near misses need the closest
* one even when another already matched. Ties go to the earliest reference.
* A nearly blank thumbnail (fp.low) looks like countless unrelated ones, so it only matches a
* reference marked `blank`: one you made by picture from a nearly blank thumbnail, after a warning.
*/
function matchPrepared(fp, db, allowSet, cfg, board) {
let bestI = -1, bestA = 0, bestD = 0, matchI = -1, matchA = 0, matchD = 0;
const rA = db.ruleA, rD = db.ruleD, rE = db.ruleE, rM = db.ruleM, ri = db.ruleIdx;
rA[0] = cfg.a; rD[0] = cfg.d; rM[0] = cfg.margin; rE[0] = cfg.mode === 'either' ? 1 : 0;
if (fp.algo === db.algo) {
const idx = boardIndex(db, board), w = db.words;
const valid = HEX64.test(fp.a) && HEX64.test(fp.d);
const a0 = valid ? parseInt(fp.a.slice(0, 8), 16) : 0, a1 = valid ? parseInt(fp.a.slice(8), 16) : 0;
const d0 = valid ? parseInt(fp.d.slice(0, 8), 16) : 0, d1 = valid ? parseInt(fp.d.slice(8), 16) : 0;
let bestSum = Infinity, matchSum = Infinity;
const low = !!fp.low, seeds = db.seeds;
for (let k = 0; k < idx.length; k++) {
const i = idx[k], o = i * 4, r = ri[i];
const da = valid ? popcount32(w[o] ^ a0) + popcount32(w[o + 1] ^ a1) : 999;
const dd = valid ? popcount32(w[o + 2] ^ d0) + popcount32(w[o + 3] ^ d1) : 999;
const sum = da + dd;
if (sum < bestSum) { bestSum = sum; bestI = i; bestA = da; bestD = dd; }
if ((rE[r] ? (da <= rA[r] || dd <= rD[r]) : (da <= rA[r] && dd <= rD[r])) && sum < matchSum && (!low || seeds[i].blank === true)) {
matchSum = sum; matchI = i; matchA = da; matchD = dd;
}
}
}
const ruleOf = i => (ri[i] ? db.rules[ri[i]] : cfg);
const best = bestI < 0 ? null : { seed: db.seeds[bestI], da: bestA, dd: bestD, rule: ruleOf(bestI) };
let match = matchI < 0 ? null : matchI === bestI ? best : { seed: db.seeds[matchI], da: matchA, dd: matchD, rule: ruleOf(matchI) };
const allowed = allowSet.has(`${fp.a} ${fp.d}`);
if (allowed) match = null;
// Whitelisted images are a decision already made, not a near miss. The margin is the closest
// reference's own (its category's, or the general one).
const br = best && best.rule;
const near = !match && !allowed && !!best && best.da <= br.a + br.margin && best.dd <= br.d + br.margin;
return { match, best, near, allowed, low: !!fp.low };
}
/** One-off comparison without a prepared database (tests, small lists). */
function evaluate(fp, seeds, allow, cfg, board) {
return matchPrepared(fp, prepareReferences(seeds, fp.algo), prepareAllow(allow), cfg, board);
}
// ---- Extended matching: cropped, bordered, mirrored and combined pictures ----
// Works with existing box-v1 references (their two 64-bit hashes); nothing new is stored.
// A reference's 8x8 (aHash) and 9x8 (dHash) grid is laid over the incoming picture at a trial
// position and size, and each cell's box-v1 average is read from a summed-area table, so at the
// right placement the cells are what the reference's own picture gave. Position and size are then
// adjusted to fit best ("alignment"). The grid may stick out past the picture (when the incoming
// picture is the tighter crop): only bits whose cells are mostly inside are compared, and
// distances are given per 64 compared bits. Runs only for pictures with no ordinary match.
/** Greyscale bytes, box-downscaled to at most 96 px on the longer side: kept per post for re-checks. */
function grayCopy(gray, w, h) {
const s = Math.min(1, 96 / Math.max(w, h)), tw = Math.max(1, Math.round(w * s)), th = Math.max(1, Math.round(h * s));
return { g: Uint8ClampedArray.from(tw === w && th === h ? gray : boxResize(gray, w, h, tw, th)), w: tw, h: th };
}
/** Summed-area table of a grayCopy, optionally of its mirror image (flip 1) or upside down (flip 2). */
function integral(px, flip = 0, invert = false) {
const { g, w, h } = px, W = w + 1, S = new Float64Array(W * (h + 1));
for (let y = 0; y < h; y++) {
const sy = flip === 2 ? h - 1 - y : y;
let row = 0;
for (let x = 0; x < w; x++) {
const v = g[sy * w + (flip === 1 ? w - 1 - x : x)];
row += invert ? 255 - v : v;
S[(y + 1) * W + x + 1] = S[y * W + x + 1] + row;
}
}
return { S, w, h };
}
const GX = new Float64Array(13), GY = new Float64Array(13), GP = new Float64Array(169); // grids up to 12 x 12
const GXI = new Int32Array(13), GXF = new Float64Array(13);
const M8 = new Float64Array(64), V8 = new Float64Array(64), M9 = new Float64Array(72), V9 = new Float64Array(72), RANK = new Float64Array(64);
/** Average and visible fraction of each cell of an nx x ny grid spanning [x0,x1] x [y0,y1]. */
function gridCells(T, x0, y0, x1, y1, nx, ny, mean, vis) {
const { S, w, h } = T, W = w + 1, cw = (x1 - x0) / nx, ch = (y1 - y0) / ny;
for (let i = 0; i <= nx; i++) {
const x = x0 + i * cw, c = x < 0 ? 0 : x > w ? w : x, xi = Math.min(w - 1, Math.floor(c));
GX[i] = c; GXI[i] = xi; GXF[i] = c - xi;
}
for (let j = 0; j <= ny; j++) { const y = y0 + j * ch; GY[j] = y < 0 ? 0 : y > h ? h : y; }
for (let j = 0; j <= ny; j++) { // the table between pixel corners is bilinear: exact box sums
const y = GY[j], yi = Math.min(h - 1, Math.floor(y)), fy = y - yi, row = yi * W, pj = j * (nx + 1);
for (let i = 0; i <= nx; i++) {
const fx = GXF[i], o = row + GXI[i];
const a = S[o], b = S[o + 1], c = S[o + W], d = S[o + W + 1];
GP[pj + i] = a + (b - a) * fx + (c - a) * fy + (a - b - c + d) * fx * fy;
}
}
const full = cw * ch;
for (let j = 0; j < ny; j++) for (let i = 0; i < nx; i++) {
const area = (GX[i + 1] - GX[i]) * (GY[j + 1] - GY[j]), k = j * nx + i, p = j * (nx + 1) + i;
vis[k] = area / full;
mean[k] = area > 0 ? (GP[p + nx + 2] - GP[p + 1] - GP[p + nx + 1] + GP[p]) / area : 0;
}
}
/** box-v1 hash of a rectangle of the picture, as the four 32-bit words prepareReferences() uses. */
function windowWords(T, b) {
const out = new Uint32Array(4);
gridCells(T, b.x, b.y, b.x + b.w, b.y + b.h, 8, 8, M8, V8);
gridCells(T, b.x, b.y, b.x + b.w, b.y + b.h, 9, 8, M9, V9);
let avg = 0;
for (let i = 0; i < 64; i++) avg += M8[i];
avg /= 64;
for (let i = 0; i < 64; i++) {
if (M8[i] > avg) out[i >> 5] |= 1 << (31 - (i & 31));
const p = (i >> 3) * 9 + (i & 7);
if (M9[p + 1] > M9[p]) out[2 + (i >> 5)] |= 1 << (31 - (i & 31));
}
return out;
}
const EXT_VISIBLE = 0.65; // a cell is compared when this much of it is inside the picture
const EXT_MIN_BITS = 36; // fewer comparable bits than this (of 64) is not evidence enough
const EXT_FLAT = 4; // dHash neighbours closer than this (0-255) are flat: their bit is weak evidence
const bitAt = (words, o, i) => (words[o + (i >> 5)] >>> (31 - (i & 31))) & 1;
/** The k-th smallest of a[0..n-1] (0-based), as a sort would give it; reorders a. */
function kthSmallest(a, n, k) {
let lo = 0, hi = n - 1;
while (lo < hi) {
const pivot = a[(lo + hi) >> 1];
let i = lo, j = hi;
while (i <= j) {
while (a[i] < pivot) i++;
while (a[j] > pivot) j--;
if (i <= j) { const t = a[i]; a[i] = a[j]; a[j] = t; i++; j--; }
}
if (k <= j) hi = j; else if (k >= i) lo = i; else return a[k];
}
return a[k];
}
/**
* Reference grid (hash words at `o`) placed at [x0,x1] x [y0,y1] over the picture: mismatching
* bits per 64 compared (a64, d64), how many were compared (na, nd), and `info`, the compared dHash
* bits that are not flat. The aHash threshold for the visible cells is set so that as many of them
* are "on" as in the reference, which needs no mean of the cells that are not visible.
*/
function gridDistance(T, x0, y0, x1, y1, words, o, r) {
gridCells(T, x0, y0, x1, y1, 9, 8, M9, V9);
let nd = 0, dd = 0, info = 0;
for (let i = 0; i < 64; i++) {
const p = (i >> 3) * 9 + (i & 7);
if (V9[p] < EXT_VISIBLE || V9[p + 1] < EXT_VISIBLE) continue;
const diff = M9[p + 1] - M9[p];
nd++;
if ((diff > 0 ? 1 : 0) !== bitAt(words, o + 2, i)) dd++;
if (diff > EXT_FLAT || diff < -EXT_FLAT) info++;
}
gridCells(T, x0, y0, x1, y1, 8, 8, M8, V8);
let na = 0, on = 0, da = 0;
for (let i = 0; i < 64; i++) if (V8[i] >= EXT_VISIBLE) { RANK[na++] = M8[i]; on += bitAt(words, o, i); }
if (na) {
const thr = on ? kthSmallest(RANK, na, na - on) : Infinity;
for (let i = 0; i < 64; i++) if (V8[i] >= EXT_VISIBLE && (M8[i] >= thr ? 1 : 0) !== bitAt(words, o, i)) da++;
}
r.na = na; r.nd = nd; r.info = info;
r.a64 = na ? da * 64 / na : 64; r.d64 = nd ? dd * 64 / nd : 64;
r.s = na < EXT_MIN_BITS || nd < EXT_MIN_BITS ? 999 : r.a64 + r.d64;
return r;
}
const MOVES = [[1, 0, 0], [-1, 0, 0], [0, 1, 0], [0, -1, 0], [0, 0, 2], [0, 0, -2]];
/**
* Best placement of reference `o` over `box` of the picture, keeping the reference's aspect ratio
* (height / width): coordinate descent over centre and width in steps of 4, 2 and 1% of the width,
* from the placements that fill the box's width and its height. The width stays within half to
* twice the box's. Stops after the coarse step when the fit is worse than `giveUp`.
* Returns {cx, cy, W, a64, d64, na, nd, info, s, visible: share of the reference inside the
* picture, cover: share of the picture it covers, evals}.
*/
function alignReference(T, box, words, o, aspect, giveUp) {
let best = null, evals = 0;
const at = (cx, cy, W) => {
if (W < box.w * 0.5 || W > box.w * 2 || W < 6) return null;
evals++;
const H = W * aspect;
return gridDistance(T, cx - W / 2, cy - H / 2, cx + W / 2, cy + H / 2, words, o, { cx, cy, W });
};
const cx0 = box.x + box.w / 2, cy0 = box.y + box.h / 2, starts = [box.w];
if (Math.abs(box.h / aspect - box.w) > 0.02 * box.w) starts.push(box.h / aspect);
for (const W0 of starts) {
let cur = at(cx0, cy0, W0);
if (!cur) continue;
for (const f of [0.04, 0.02, 0.01]) {
for (let it = 0, moved = true; moved && it < 12; it++) {
moved = false;
const st = f * cur.W;
for (const [dx, dy, dw] of MOVES) {
const c = at(cur.cx + dx * st, cur.cy + dy * st, cur.W + dw * st);
if (c && c.s < cur.s) { cur = c; moved = true; }
}
}
if (cur.s > giveUp) break;
}
if (!best || cur.s < best.s) best = cur;
}
if (!best) return null;
const H = best.W * aspect, x0 = best.cx - best.W / 2, y0 = best.cy - H / 2;
const inside = Math.max(0, Math.min(T.w, x0 + best.W) - Math.max(0, x0)) * Math.max(0, Math.min(T.h, y0 + H) - Math.max(0, y0));
best.visible = inside / (best.W * H);
best.cover = inside / (T.w * T.h);
best.evals = evals;
return best;
}
const DETAIL = 12, DM = new Float64Array(DETAIL * DETAIL), DV = new Float64Array(DETAIL * DETAIL);
/** A reference's detail: its picture (or a box of it) as 12 x 12 grey cells, stored as base64 text. */
function detailOf(T, box = { x: 0, y: 0, w: T.w, h: T.h }) {
gridCells(T, box.x, box.y, box.x + box.w, box.y + box.h, DETAIL, DETAIL, DM, DV);
let bin = '';
for (const v of DM) bin += String.fromCharCode(Math.round(v));
return btoa(bin);
}
const DETAIL_TEXT = /^[A-Za-z0-9+/]{192}$/;
const detailBytes = t => (DETAIL_TEXT.test(t || '') ? Uint8Array.from(atob(t), c => c.charCodeAt(0)) : null);
/**
* Correlation (-1..1) between a reference's detail and the picture's cells at the placement an
* alignment found, over the cells that are mostly inside the picture (null if too few are).
* Far more evidence than the 128 hash bits: unrelated pictures that happen to share a hash
* layout (a figure on a plain background...) differ in these grey levels.
*/
function detailCorrelation(T, cx, cy, W, aspect, detail) {
const H = W * aspect;
gridCells(T, cx - W / 2, cy - H / 2, cx + W / 2, cy + H / 2, DETAIL, DETAIL, DM, DV);
let n = 0, sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0;
for (let k = 0; k < DETAIL * DETAIL; k++) {
if (DV[k] < EXT_VISIBLE) continue;
const x = DM[k], y = detail[k];
n++; sx += x; sy += y; sxx += x * x; syy += y * y; sxy += x * y;
}
if (n < DETAIL * DETAIL * 0.5) return null;
const vx = sxx - sx * sx / n, vy = syy - sy * sy / n;
return vx > 0 && vy > 0 ? (sxy - sx * sy / n) / Math.sqrt(vx * vy) : 0;
}
/**
* An ordinary match near the limits (aHash + dHash distance ORD_SUM or more) with a reference that
* has a detail must also look like it: the whole picture's 12x12 cells must correlate with the detail
* at least ORD_DETAIL. Pictures that share only a hash layout (a small figure on white...) fail this.
* Returns the correlation, or null when there is nothing to check.
*/
const ORD_SUM = 7, ORD_DETAIL = 0.85;
function ordinaryDetail(px, db, m) {
if (!px || !m || m.how || m.da + m.dd < ORD_SUM || px.w < 12 || px.h < 12) return null;
const i = db.seeds.indexOf(m.seed);
if (i < 0) return null;
if (!db.details.has(i)) db.details.set(i, detailBytes(m.seed.detail));
const det = db.details.get(i);
if (!det) return null;
const T = integral(px);
return detailCorrelation(T, T.w / 2, T.h / 2, T.w, T.h / T.w, det);
}
/**
* Retouched copies: drawn over, darkened, recoloured, made grainy. Lines and tints change many dHash
* bits but few aHash bits, and hardly the 12x12 detail (a correlation ignores brightness and
* contrast). So a whole picture whose aHash is within `rules.a` of a reference's (and its own aHash
* limit) and whose cells correlate with the reference's detail at least `rules.corr` matches although
* its dHash is over the limit. Only references with a detail; a reference judged by dHash 0 never.
* LOOK is set above what different shots of one scene reach; CLOSE (opt-in "close variants") below.
* Returns {seed, da, dd, corr, rule} for the best correlation, or null.
*/
// LOOK: retouched copies (the same picture drawn over or recoloured); CLOSE: the opt-in close
// variants. Both need the picture's detail to agree, so pictures that only share a layout don't match.
const LOOK = { a: 2, d: 20, corr: 0.96 }, CLOSE = { a: 4, d: 16, corr: 0.93 };
function matchLookalike(px, fp, db, cfg, board, rules = LOOK) {
if (!px || !fp || fp.low || px.w < 12 || px.h < 12 || !HEX64.test(fp.a) || !HEX64.test(fp.d)) return null;
const idx = boardIndex(db, board), w = db.words;
const a0 = parseInt(fp.a.slice(0, 8), 16), a1 = parseInt(fp.a.slice(8), 16), d0 = parseInt(fp.d.slice(0, 8), 16), d1 = parseInt(fp.d.slice(8), 16);
let T = null, best = null;
for (let k = 0; k < idx.length; k++) {
const i = idx[k], o = i * 4, r = db.ruleIdx[i] ? db.rules[db.ruleIdx[i]] : cfg;
if (!(rules === CLOSE ? db.closeOK : db.extOK)[i]) continue; // switched off for this reference
const da = popcount32(w[o] ^ a0) + popcount32(w[o + 1] ^ a1);
if (da > Math.min(rules.a, r.a) || r.d === 0) continue;
const dd = popcount32(w[o + 2] ^ d0) + popcount32(w[o + 3] ^ d1);
if (dd > rules.d) continue;
const seed = db.seeds[i];
if (!db.details.has(i)) db.details.set(i, detailBytes(seed.detail));
const det = db.details.get(i);
if (!det) continue;
T = T || integral(px);
const corr = detailCorrelation(T, T.w / 2, T.h / 2, T.w, T.h / T.w, det);
if (corr >= rules.corr && (!best || corr > best.corr)) best = { seed, da, dd, corr, rule: r };
}
return best;
}
/** Content inside near-uniform margins (any one shade per side, JPEG noise allowed), or null. */
function contentBox(px) {
const { g, w, h } = px, TOL = 12;
const flat = (n, at, v) => { let bad = 0; for (let k = 0; k < n; k++) if (Math.abs(at(k) - v) > TOL && ++bad > n * 0.03) return false; return true; };
let t = 0, b = h, l = 0, r = w;
for (const v of [g[0], g[w * h - 1], g[w - 1], g[w * (h - 1)]]) {
while (b - t > 8 && flat(r - l, k => g[t * w + l + k], v)) t++;
while (b - t > 8 && flat(r - l, k => g[(b - 1) * w + l + k], v)) b--;
while (r - l > 8 && flat(b - t, k => g[(t + k) * w + l], v)) l++;
while (r - l > 8 && flat(b - t, k => g[(t + k) * w + r - 1], v)) r--;
}
const kept = (r - l) * (b - t) / (w * h);
return kept < 0.92 && r - l >= 12 && b - t >= 12 ? { x: l, y: t, w: r - l, h: b - t } : null;
}
/**
* The picture on a canvas whose margins are not plain (lines drawn across them): the longest run of
* rows, then of columns, where most pixels differ from the canvas shade (the median of the outer edge).
* A drawn line crosses only a row or two, so it can't make a long run. Null if that is the whole picture.
*/
function blockBox(px) {
const { g, w, h } = px, edge = [];
for (let x = 0; x < w; x++) edge.push(g[x], g[(h - 1) * w + x]);
for (let y = 0; y < h; y++) edge.push(g[y * w], g[y * w + w - 1]);
const shade = edge.sort((x, y) => x - y)[edge.length >> 1];
const longest = (n, ok) => {
let best = [0, -1], start = -1;
for (let i = 0; i <= n; i++) {
if (i < n && ok(i)) { if (start < 0) start = i; } else if (start >= 0) { if (i - start > best[1] - best[0] + 1) best = [start, i - 1]; start = -1; }
}
return best;
};
// rows (then columns) where at least 40% of the pixels differ from the canvas shade by more than 24
const [t, b] = longest(h, y => { let n = 0; for (let o = y * w, x = 0; x < w; x++) if (Math.abs(g[o + x] - shade) > 24) n++; return n >= 0.4 * w; });
if (b - t + 1 < 12) return null;
const [l, r] = longest(w, x => { let n = 0; for (let y = t; y <= b; y++) if (Math.abs(g[y * w + x] - shade) > 24) n++; return n >= 0.4 * (b - t + 1); });
const box = { x: l, y: t, w: r - l + 1, h: b - t + 1 };
return box.w >= 12 && box.w * box.h < 0.85 * w * h ? box : null;
}
/**
* The picture inside an added band of coloured stripes: rows
* (or columns) at an edge that are each one flat shade while neighbouring ones keep changing. A plain
* wall or sky is flat but doesn't alternate; plain borders are trimmed anyway (contentBox()). Needs a
* band of at least 15% and leaves at least half the picture. Stored with a reference when it is
* blacklisted ("part"), so copies without the band, or with different stripes, still match.
*/
function pictureOf(px) {
const { g, w, h } = px;
// [mean, flat] of row y (or column x when `col`)
const line = (i, col) => {
const n = col ? h : w;
let s = 0, ss = 0;
for (let k = 0; k < n; k++) { const v = col ? g[k * w + i] : g[i * w + k]; s += v; ss += v * v; }
const m = s / n;
return [m, Math.sqrt(Math.max(0, ss / n - m * m)) <= 12];
};
// length of the striped band from one edge inwards: flat lines, at least 1 in 5 changing shade by > 20
const band = (n, col, fromEnd) => {
let len = 0, changes = 0, prev = null;
for (let k = 0; k < n; k++) {
const [m, flat] = line(fromEnd ? n - 1 - k : k, col);
if (!flat) break;
if (prev !== null && Math.abs(m - prev) > 20) changes++;
prev = m;
len++;
}
return len >= 0.15 * n && changes >= len / 5 ? len : 0;
};
const top = band(h, false, false), bottom = band(h, false, true), left = band(w, true, false), right = band(w, true, true);
if (!(top || bottom || left || right)) return null;
const box = { x: left, y: top, w: w - left - right, h: h - top - bottom };
if (box.w * box.h < 0.5 * w * h || box.w < 12 || box.h < 12) return null;
const sub = new Float64Array(box.w * box.h);
for (let y = 0; y < box.h; y++) for (let x = 0; x < box.w; x++) sub[y * box.w + x] = g[(box.y + y) * w + box.x + x];
const fp = hashGray(sub, box.w, box.h);
return fp.low ? null : { a: fp.a, d: fp.d, w: box.w, h: box.h, detail: detailOf(integral(px), box) };
}
/**
* Pieces of a reference's picture: its top, bottom, left and right halves, its centre 60%,
* and its top, bottom, left and right 70%, each kept as the hashes and detail of that window. A heavily
* cropped copy (the bottom half of a photo, say) is then an ordinary or edited copy of one piece; a
* match reports the reference. Stored with a reference when it is blacklisted (one without them gets
* them when its exact image is seen again); nearly blank pieces (a plain sky or wall) are left out.
* w/h are in the reference thumbnail's pixels (fp), like the reference's own size.
*/
const PIECES = [[0, 0, 1, 0.5], [0, 0.5, 1, 0.5], [0, 0, 0.5, 1], [0.5, 0, 0.5, 1], [0.2, 0.2, 0.6, 0.6],
[0, 0, 1, 0.7], [0, 0.3, 1, 0.7], [0, 0, 0.7, 1], [0.3, 0, 0.7, 1]];
function piecesOf(px, fp) {
if (!px || !fp || px.w < 24 || px.h < 24) return [];
const T = integral(px), { g, w, h } = px, out = [];
const hex = (u, v) => u.toString(16).padStart(8, '0') + v.toString(16).padStart(8, '0');
for (const [bx, by, bw, bh] of PIECES) {
const box = { x: bx * w, y: by * h, w: bw * w, h: bh * h }, q = windowWords(T, box);
// nearly blank? the window's own pixels, as a picture of their own
const x0 = Math.round(box.x), y0 = Math.round(box.y), sw = Math.round(box.w), sh = Math.round(box.h), sub = new Float64Array(sw * sh);
for (let y = 0; y < sh; y++) for (let x = 0; x < sw; x++) sub[y * sw + x] = g[(y0 + y) * w + x0 + x];
if (hashGray(sub, sw, sh).low) continue;
out.push({ a: hex(q[0], q[1]), d: hex(q[2], q[3]), w: Math.round(bw * fp.w), h: Math.round(bh * fp.h), detail: detailOf(T, box) });
}
return out;
}
/**
* Pieces as stored: one text "p1:" + base64 of 92 bytes per piece: aHash and dHash (8 bytes
* each), width and height (2 bytes each), and the detail at 16 grey levels (144 cells, 2 per byte).
* About 1.1 KB for 9 pieces.
*/
const PIECE_BYTES = 92, PIECE_TEXT = /^p1:[A-Za-z0-9+/]+={0,2}$/;
function packPieces(list) {
let bin = '';
for (const pc of list) {
const det = detailBytes(pc.detail);
if (!det || !HEX64.test(pc.a) || !HEX64.test(pc.d) || !(pc.w > 0 && pc.w < 65536) || !(pc.h > 0 && pc.h < 65536)) continue;
for (const hex of [pc.a, pc.d]) for (let i = 0; i < 16; i += 2) bin += String.fromCharCode(parseInt(hex.slice(i, i + 2), 16));
bin += String.fromCharCode(pc.w >> 8, pc.w & 255, pc.h >> 8, pc.h & 255);
for (let k = 0; k < 144; k += 2) bin += String.fromCharCode((Math.round(det[k] / 17) << 4) | Math.round(det[k + 1] / 17));
}
return bin ? 'p1:' + btoa(bin) : null;
}
/** Stored pieces -> [{a, d, w, h, detail (base64, 256 levels)}]; [] for anything malformed. */
function unpackPieces(text) {
if (typeof text !== 'string' || text.length > 3 + Math.ceil(PIECES.length * PIECE_BYTES / 3) * 4 || !PIECE_TEXT.test(text)) return [];
let bin;
try { bin = atob(text.slice(3)); } catch (e) { return []; }
if (!bin.length || bin.length % PIECE_BYTES || bin.length / PIECE_BYTES > PIECES.length) return [];
const out = [], hex = (o, n) => { let s = ''; for (let i = 0; i < n; i++) s += bin.charCodeAt(o + i).toString(16).padStart(2, '0'); return s; };
for (let o = 0; o < bin.length; o += PIECE_BYTES) {
let det = '';
for (let k = 0; k < 72; k++) { const v = bin.charCodeAt(o + 20 + k); det += String.fromCharCode((v >> 4) * 17, (v & 15) * 17); }
out.push({ a: hex(o, 8), d: hex(o + 8, 8), w: bin.charCodeAt(o + 16) << 8 | bin.charCodeAt(o + 17), h: bin.charCodeAt(o + 18) << 8 | bin.charCodeAt(o + 19), detail: btoa(det) });
}
return out;
}
/** Stored pieces text if it is valid, else null. */
const piecesText = v => (typeof v === 'string' && unpackPieces(v).length ? v : null);
const pieceCount = v => (typeof v === 'string' ? Math.round((v.length - 3) * 3 / 4 / PIECE_BYTES) : 0);
/**
* Panels of a side-by-side or stacked composite. A join between two pictures is a straight line
* where, on most rows (or columns), the brightness step is the largest one nearby; edges inside a
* photo are rarely that straight and complete. Up to two joins per direction, the strongest first,
* at least 15% from the edges and from each other; each panel is then trimmed of uniform margins.
* Panels only add candidates, which have to pass the strict panel rules. minScore: the share of rows
* (columns) that must step at a join.
*/
function panelBoxes(px, minScore = 0.5) {
const { g, w, h } = px, out = [];
// Joins across n positions, over m rows (or columns): position k of line j is g[j * stride + k * step].
// Each line's steps are read once, as they slide past.
const cuts = (n, m, step, stride) => {
const peaks = new Float64Array(n + 1), found = [];
if (n < 4) return found;
for (let j = 0; j < m; j++) {
const o = j * stride;
// steps into k - 1 (before), k (s) and k + 1 (after)
let before = Math.abs(g[o + step] - g[o]), s = Math.abs(g[o + 2 * step] - g[o + step]);
for (let k = 2; k < n - 1; k++) {
const after = Math.abs(g[o + (k + 1) * step] - g[o + k * step]);
if (s >= 12 && s > before && s >= after) peaks[k]++;
before = s;
s = after;
}
}
const gap = Math.max(3, Math.round(n * 0.15)), score = [];
for (let k = gap; k <= n - gap; k++) score.push([(peaks[k - 1] + peaks[k] + peaks[k + 1]) / m, k]);
score.sort((x, y) => y[0] - x[0]);
for (const [v, k] of score) {
if (v < minScore || found.length === 2) break;
if (found.every(c => Math.abs(c - k) >= gap)) found.push(k);
}
return found.sort((x, y) => x - y);
};
const add = b => {
const sub = { g: new Uint8ClampedArray(b.w * b.h), w: b.w, h: b.h };
for (let y = 0; y < b.h; y++) sub.g.set(g.subarray((b.y + y) * w + b.x, (b.y + y) * w + b.x + b.w), y * b.w);
const c = contentBox(sub);
out.push(c ? { x: b.x + c.x, y: b.y + c.y, w: c.w, h: c.h } : b);
};
const xs = cuts(w, h, 1, w);
if (xs.length) [0, ...xs].forEach((x, i, a) => add({ x, y: 0, w: (a[i + 1] || w) - x, h }));
const ys = cuts(h, w, w, 1);
if (ys.length) [0, ...ys].forEach((y, i, a) => add({ x: 0, y, w, h: (a[i + 1] || h) - y }));
return out.filter(b => b.w >= 12 && b.h >= 12);
}
// Inner parts tried as pictures of their own (share of each side), and the tilts turned back (degrees).
const ZOOM_KEEP = [0.8, 0.65], TILT_DEGREES = [-3, -1.5, 1.5, 3], TILT_KEEP = 0.85;
/** A greyscale copy turned clockwise by `q` quarter turns. */
function quarterTurn(px, q) {
const { g, w, h } = px, W = q === 2 ? w : h, H = q === 2 ? h : w, o = new Uint8ClampedArray(w * h);
for (let y = 0; y < H; y++) for (let x = 0; x < W; x++) {
const sx = q === 1 ? y : q === 2 ? w - 1 - x : w - 1 - y, sy = q === 1 ? h - 1 - x : q === 2 ? h - 1 - y : x;
o[y * W + x] = g[sy * w + sx];
}
return { g: o, w: W, h: H };
}
/** The centred part of a greyscale copy, `keep` of each side, as a picture of its own (null if too small). */
function innerPicture(px, keep) {
const w = Math.round(px.w * keep), h = Math.round(px.h * keep), x0 = Math.floor((px.w - w) / 2), y0 = Math.floor((px.h - h) / 2);
if (w < 12 || h < 12) return null;
const g = new Uint8ClampedArray(w * h);
for (let y = 0; y < h; y++) g.set(px.g.subarray((y0 + y) * px.w + x0, (y0 + y) * px.w + x0 + w), y * w);
return { g, w, h };
}
/**
* A greyscale copy turned by `deg` degrees about its centre (bilinear), cut to the largest centred
* rectangle of its shape that has no empty corners, then to `keep` of each side; null if too small.
*/
function turnedPicture(px, deg, keep) {
const a = deg * Math.PI / 180, c = Math.cos(a), s = Math.abs(Math.sin(a)), { w: sw, h: sh, g: src } = px;
const k = keep * Math.min(sw / (sw * c + sh * s), sh / (sw * s + sh * c));
const w = Math.round(sw * k), h = Math.round(sh * k);
if (w < 12 || h < 12) return null;
const g = new Uint8ClampedArray(w * h), sin = Math.sin(a), cx = sw / 2, cy = sh / 2;
for (let y = 0; y < h; y++) {
const dy = y + 0.5 - h / 2;
for (let x = 0; x < w; x++) {
const dx = x + 0.5 - w / 2;
const fx = Math.min(sw - 1, Math.max(0, cx + dx * c - dy * sin - 0.5)), fy = Math.min(sh - 1, Math.max(0, cy + dx * sin + dy * c - 0.5));
const xi = Math.min(sw - 2, Math.floor(fx)), yi = Math.min(sh - 2, Math.floor(fy)), tx = fx - xi, ty = fy - yi, o = yi * sw + xi;
g[y * w + x] = src[o] * (1 - tx) * (1 - ty) + src[o + 1] * tx * (1 - ty) + src[o + sw] * (1 - tx) * ty + src[o + sw + 1] * tx * ty;
}
}
return { g, w, h };
}
/**
* Rules per way of matching (distances per 64 compared bits; s: aHash + dHash). The aHash and
* dHash limits of the reference's own rules (its category's, or the general ones) are a ceiling. `visible`: share of the reference that
* must be inside the picture; `cover`: share of the picture it must cover; `info`: compared dHash
* bits that are not flat (pictures that are mostly plain background match too easily). `gate`:
* how close (aHash + dHash bits) a window's own hash must be to a reference for it to be aligned
* at all, which keeps alignments few.
* Calibrated on thousands of unrelated thumbnails: each is one step stricter than the loosest
* rule with no unrelated match there.
* A reference with a stored detail (detailOf) is held to that instead: its hash limits are 2 bits
* looser (s: 5; still within the ceiling) but the picture at the found placement must correlate
* with it by EXT_DETAIL (or the way's own `corr`).
* Without it, hashes alone get ambiguous with many references (tens of thousands).
*/
const EXT_RULES = {
crop: { gate: 36, a: 4, d: 5, s: 5, visible: 0.7, cover: 0.5, info: 20 },
border: { gate: 28, a: 4, d: 5, s: 6, visible: 0.85, cover: 0.08, info: 24 },
mirror: { gate: 30, a: 2, d: 3, s: 4, visible: 0.7, cover: 0.5, info: 24 },
flip: { gate: 12, a: 2, d: 3, s: 3, visible: 0.7, cover: 0.5, info: 24 },
panel: { gate: 28, a: 2, d: 4, s: 5, visible: 0.85, cover: 0.15, info: 32 },
invert: { gate: 36, a: 4, d: 5, s: 5, visible: 0.7, cover: 0.5, info: 20 },
zoom: { gate: 40, a: 4, d: 5, s: 9, visible: 0.7, cover: 0.85, info: 24, corr: 0.96 },
tilt: { gate: 28, a: 4, d: 5, s: 9, visible: 0.7, cover: 0.8, info: 32, corr: 0.96 },
turn: { gate: 30, a: 2, d: 3, s: 4, visible: 0.7, cover: 0.5, info: 24 },
};
// Candidates aligned per window (the closest by hash first): fewer for the many inner-part and tilt windows.
const EXT_PICK = { zoom: 2, tilt: 1, turn: 1 };
const EXT_PER_WINDOW = 6, EXT_DETAIL = 0.93;
// Detail correlation where an alignment would start, below which it is not run.
let EXT_START = 0.6;
// Inner parts and tilts start further from where they end (a closer crop, a turn), so a lower bar.
const EXT_START_FOR = { zoom: 0.4, tilt: 0.6 };
/**
* Extended matching of a picture (grayCopy) with no ordinary match, against the references that
* apply on `board`. opts: {crop, border, mirror, panels, invert, zoom, tilt, turn} (which ways are on). Each window (whole
* picture, its content inside borders, its mirror image / upside-down, each panel) picks its few
* closest references by its own hash, within its way's gate; each is aligned and must pass its way's
* rules. Returns {seed, how, a64, d64, info, visible, cover, corr (null without detail), rule} for the best fit of the first
* window that has one, or null. stats (optional) counts windows and alignments.
*/
function matchExtended(px, db, cfg, board, opts, stats) {
const idx = boardIndex(db, board), w = db.words;
if (!idx.length || px.w < 12 || px.h < 12) return null;
const T = integral(px), wins = [], full = { x: 0, y: 0, w: px.w, h: px.h };
if (opts.crop) wins.push({ how: 'crop', T, box: full });
if (opts.border) {
const b = contentBox(px), bb = blockBox(px);
if (b) wins.push({ how: 'border', T, box: b });
// a canvas with lines drawn across its margins: the plain-border trim stops at them
if (bb && (!b || Math.abs(bb.w - b.w) + Math.abs(bb.h - b.h) > 4)) wins.push({ how: 'border', T, box: bb });
}
if (opts.invert) wins.push({ how: 'invert', T: integral(px, 0, true), box: full });
if (opts.mirror) for (const flip of [1, 2]) wins.push({ how: 'mirror', flip, T: integral(px, flip), box: full });
if (opts.panels) for (const b of panelBoxes(px)) wins.push({ how: 'panel', T, box: b });
// and each quarter of the picture, as a panel of a 2x2 collage
if (opts.panels) for (const [x, y] of [[0, 0], [0.5, 0], [0, 0.5], [0.5, 0.5]]) wins.push({ how: 'panel', quarter: true, T, box: { x: x * px.w, y: y * px.h, w: px.w / 2, h: px.h / 2 } });
// A copy turned by a quarter or half turn (or a quarter turn and mirrored): turned back.
if (opts.turn) for (const [q, m] of [[1, 0], [2, 0], [3, 0], [1, 1], [3, 1]]) {
const sub = quarterTurn(px, q);
wins.push({ how: 'turn', T: integral(sub, m), box: { x: 0, y: 0, w: sub.w, h: sub.h } });
}
// Inner parts of the picture, each as a picture of its own (also mirrored). Small turns: turned back,
// cut to the part with no empty corners.
const own = sub => sub && { T: integral(sub), box: { x: 0, y: 0, w: sub.w, h: sub.h } };
if (opts.zoom) for (const keep of ZOOM_KEEP) {
const sub = innerPicture(px, keep);
if (sub) for (const flip of [0, 1]) wins.push({ how: 'zoom', T: integral(sub, flip), box: { x: 0, y: 0, w: sub.w, h: sub.h } });
}
if (opts.tilt) for (const deg of TILT_DEGREES) { const o = own(turnedPicture(px, deg, TILT_KEEP)); if (o) wins.push({ how: 'tilt', ...o }); }
for (const win of wins) { win.q = windowWords(win.T, win.box); win.cand = []; win.gate = EXT_RULES[win.flip === 2 ? 'flip' : win.how].gate; win.pick = EXT_PICK[win.how] || EXT_PER_WINDOW; }
// rotated, mirrored, inverted and quarter windows are compared with whole references only (not their main pictures or pieces)
if (!db.partOf) db.partOf = Uint8Array.from(db.seeds, s => (s.isPart ? 1 : 0));
const partOf = db.partOf, whole = Uint8Array.from(wins, win => (win.how === 'turn' || win.how === 'mirror' || win.how === 'invert' || win.quarter ? 1 : 0));
// One pass over the references for every window, keeping each window's closest few.
const n = wins.length, q = new Int32Array(n * 4), gate = new Int32Array(n);
wins.forEach((win, j) => { q.set(win.q, j * 4); gate[j] = win.gate; });
for (let k = 0; k < idx.length; k++) {
if (!db.extOK[idx[k]]) continue; // edited copies switched off for this reference
const o = idx[k] * 4, w0 = w[o], w1 = w[o + 1], w2 = w[o + 2], w3 = w[o + 3], part = partOf[idx[k]];
for (let j = 0; j < n; j++) {
if (part && whole[j]) continue;
const qj = j * 4, sa = popcount32(w0 ^ q[qj]) + popcount32(w1 ^ q[qj + 1]);
if (sa > gate[j]) continue;
const s = sa + popcount32(w2 ^ q[qj + 2]) + popcount32(w3 ^ q[qj + 3]);
if (s > gate[j]) continue;
const c = wins[j].cand;
if (c.length === wins[j].pick) { if (s >= c[c.length - 1][0]) continue; c.pop(); }
let p = c.length;
while (p > 0 && c[p - 1][0] > s) p--;
c.splice(p, 0, [s, idx[k]]);
}
}
if (stats) stats.windows += wins.length;
for (const win of wins) {
let best = null;
for (const [gate, i] of win.cand) {
const how = win.flip === 2 ? 'flip' : win.how, seed = db.seeds[i], r = db.ruleIdx[i] ? db.rules[db.ruleIdx[i]] : cfg, R = EXT_RULES[how];
if (!db.details.has(i)) db.details.set(i, detailBytes(seed.detail)); // decoded on first use
const det = db.details.get(i), loose = det ? 2 : 0;
if (!det && (how === 'zoom' || how === 'tilt')) continue; // these need the reference's detail to agree as well
const A = Math.min(R.a + loose, r.a), Dl = Math.min(R.d + loose, r.d), S = R.s + 2.5 * loose;
// The grid takes the reference picture's shape when it is known (references added here keep their
// size; imported ones don't), else the window's. When both are known and differ (a copy cropped
// in one direction only), both are tried, so added and imported references behave the same.
const own = seed.w > 0 && seed.h > 0 ? seed.h / seed.w : 0, its = win.box.h / win.box.w;
for (const aspect of own && Math.abs(own / its - 1) > 0.03 ? [own, its] : [own || its]) {
// A reference whose detail looks nothing like the picture where the search would start is not searched.
const start = EXT_START_FOR[how] != null ? EXT_START_FOR[how] : EXT_START;
if (det && start > 0) {
const c0 = detailCorrelation(win.T, win.box.x + win.box.w / 2, win.box.y + win.box.h / 2, aspect * win.box.w > win.box.h ? win.box.h / aspect : win.box.w, aspect, det);
if (c0 !== null && c0 < start) continue;
}
if (stats) stats.aligns++;
const f = alignReference(win.T, win.box, w, i * 4, aspect, 2.5 * (A + Dl));
const corr = f && det ? detailCorrelation(win.T, f.cx, f.cy, f.W, aspect, det) : null;
if (stats && f) stats.evals += f.evals;
if (stats && stats.fits && f) stats.fits.push({ i, how, gate, corr, cx: f.cx, cy: f.cy, W: f.W, aspect, a64: f.a64, d64: f.d64, info: f.info, visible: f.visible, cover: f.cover, na: f.na, nd: f.nd });
if (!f || f.a64 > A || f.d64 > Dl || f.s > S || f.info < R.info || f.visible < R.visible || f.cover < R.cover) continue;
if (det && !(corr >= (R.corr || EXT_DETAIL))) continue;
if (!best || f.s < best.s) best = { seed, how, a64: f.a64, d64: f.d64, s: f.s, info: f.info, visible: f.visible, cover: f.cover, corr, rule: r };
}
}
if (best && !(stats && stats.fits)) return best; // calibration (tests) records every window's fits
}
return null;
}
// ---- Overlapping crops: any shift, a range of sizes, small tilts
// Two crops of one picture may only partly overlap, so neither holds the other. Each reference's stored 12x12 detail is resampled to square cells (at four half-cell offsets,
// and mirrored too) and cut into 4x4-cell patterns, filed by their shape (which cells are lighter than the pattern's
// average) in an index built in memory. The picture is sampled once per size, on square cells; each of
// its patterns votes for the references and placements that hold it. The few placements with most
// votes are tuned and checked: the detail must correlate where they overlap, the hash bits there must
// agree, and the overlap must be a fair part of both. Nothing new is stored.
const OV = {
minStd: 6, // reference patterns flatter than this (grey levels) say nothing
minStdIn: 3, // the picture's: lower, as a brightened or faded copy has less contrast
sizes: [0.6, 3], // the reference's width against the picture's
steps: 12, // sizes tried between those
votes: 2, // at least this many agreeing patterns before a placement is compared
lead: 256, // references (per size and side) whose votes are then placed
shortlist: 64, // placements (most votes first) whose detail is then compared where they were voted
verify: 6, // placements tuned and checked per picture, best compared first
tilts: [-3, -1.5, 1.5, 3],
cells: 60, // detail cells that must overlap (of 144)
bits: 24, // hash bits that must be compared (of 64 each)
common: 128, // a pattern held this many times in the index (as stored and mirrored) is too common to say anything
budget: 6e6, // patterns the index holds at most (4 bytes each)
corr: 0.96, a: 8, d: 12, info: 24, // the checks at the placement (hash distances per 64 bits); calibrated like EXT_RULES
infoTurned: 32, // a tilted placement needs as much detail as the tilted-copy check asks
quarter: 0.85, // a placement over fewer than three quarters of the reference must correlate this well in each
start: 0.5, // a placement that correlates less than this as voted is not tuned
turn: 0.75, // tilts are tried only when the straight placement correlates between this and `corr`
};
// Pieces indexed too (piecesOf), besides each whole reference: closer crops are crops of one of them.
const OV_PIECES = ['centre', 'half'];
// A vote's weight in the first count by how many entries share its pattern: rarer patterns say more.
const OV_RARE = Uint8Array.from({ length: 1025 }, (_, n) => Math.max(1, Math.round(8 - Math.log2(Math.max(1, n)))));
/** The reference's shape (height / width): its stored size, else its pieces' (a half spans the whole width or height). */
function entryAspect(s) {
if (s.w > 0 && s.h > 0) return s.h / s.w;
const pcs = typeof s.pieces === 'string' ? unpackPieces(s.pieces) : [];
const W = Math.max(0, ...pcs.map(p => p.w)), H = Math.max(0, ...pcs.map(p => p.h));
return W > 0 && H > 0 ? H / W : 0;
}
/** The 16-bit shape of the 4x4 cells of `m` (row width `rw`) at (i, j), or -1 if too flat; `weak` gets its two least certain cells. */
const PATTERN_SD = [0]; // the last pattern's spread (grey levels)
function patternKey(m, rw, i, j, weak, minStd = OV.minStd) {
let sum = 0, sq = 0;
for (let y = 0; y < 4; y++) for (let x = 0; x < 4; x++) { const v = m[(j + y) * rw + i + x]; sum += v; sq += v * v; }
const mean = sum / 16, sd = Math.sqrt(Math.max(0, sq / 16 - mean * mean));
PATTERN_SD[0] = sd;
if (sd < minStd) return -1;
let key = 0, w1 = 1e9, w2 = 1e9, b1 = 0, b2 = 0;
for (let k = 0; k < 16; k++) {
const v = m[(j + (k >> 2)) * rw + i + (k & 3)] - mean, d = Math.abs(v);
if (v > 0) key |= 1 << k;
if (d < w1) { w2 = w1; b2 = b1; w1 = d; b1 = k; } else if (d < w2) { w2 = d; b2 = k; }
}
if (weak) { weak[0] = b1; weak[1] = b2; }
return key;
}
/** A 12x12 detail resampled to square cells (its width's cells; rows by its shape), shifted by (px, py) cells. */
function squareGrid(det, aspect, px, py) {
const rows = Math.floor(DETAIL * aspect - py), cols = DETAIL - (px ? 1 : 0), out = new Float64Array(Math.max(0, rows * cols));
// bilinear between cell centres, in the detail's own cell units; the columns' and rows' positions once
const XI = new Int32Array(cols), TX = new Float64Array(cols);
for (let i = 0; i < cols; i++) {
const fx = Math.min(DETAIL - 1, Math.max(0, i + 0.5 + px - 0.5));
XI[i] = Math.min(DETAIL - 2, Math.floor(fx)); TX[i] = fx - XI[i];
}
for (let j = 0; j < rows; j++) {
const fy = Math.min(DETAIL - 1, Math.max(0, (j + 0.5 + py) / aspect - 0.5)), yi = Math.min(DETAIL - 2, Math.floor(fy)), ty = fy - yi;
for (let i = 0; i < cols; i++) {
const tx = TX[i], o = yi * DETAIL + XI[i];
out[j * cols + i] = det[o] * (1 - tx) * (1 - ty) + det[o + 1] * tx * (1 - ty) + det[o + DETAIL] * (1 - tx) * ty + det[o + DETAIL + 1] * tx * ty;
}
}
return { g: out, rows, cols };
}
/**
* One entry's patterns into the build's scratch arrays (b.keys, b.offs, b.sds): its square-cell grids, as
* stored and mirrored, each at four half-cell offsets, cut into 4x4-cell patterns. Returns how many.
*/
function entryPatterns(b, det, asp) {
const mirrored = det.slice();
for (let y = 0; y < DETAIL; y++) for (let x = 0; x < DETAIL; x++) mirrored[y * DETAIL + x] = det[y * DETAIL + DETAIL - 1 - x];
const grids = [];
let cap = 0;
for (const [g, m] of [[det, 0], [mirrored, 1]]) for (const [px, py] of [[0, 0], [0.5, 0], [0, 0.5], [0.5, 0.5]]) {
const q = squareGrid(g, asp, px, py);
grids.push([q, m, px, py]);
cap += Math.max(0, q.rows - 3) * Math.max(0, q.cols - 3);
}
if (!b.keys || b.keys.length < cap) { b.keys = new Uint16Array(cap * 2); b.offs = new Uint16Array(cap * 2); b.sds = new Uint8Array(cap * 2); }
const keys = b.keys, offs = b.offs, sds = b.sds, minStd = OV.minStd;
let n = 0;
for (const [q, m, px, py] of grids) {
const G = q.g, rw = q.cols;
for (let j = 0; j + 4 <= q.rows; j++) for (let x = 0; x + 4 <= rw; x++) {
// the 4x4 pattern's shape (as patternKey)
let sum = 0, sq = 0;
for (let y = 0; y < 4; y++) for (let k = 0; k < 4; k++) { const v = G[(j + y) * rw + x + k]; sum += v; sq += v * v; }
const mean = sum / 16, sd = Math.sqrt(Math.max(0, sq / 16 - mean * mean));
if (sd < minStd) continue;
let key = 0;
for (let k = 0; k < 16; k++) if (G[(j + (k >> 2)) * rw + x + (k & 3)] - mean > 0) key |= 1 << k;
keys[n] = key; offs[n] = (2 * x + 2 * px) | ((2 * j + 2 * py) << 6) | (m << 12); sds[n] = Math.min(255, Math.round(sd)); // the corner in half cells, mirrored or not
n++;
}
}
return n;
}
/**
* The pattern index of the references that may match overlapping crops, built once per database, in two
* passes so that only the patterns kept are ever held: the first counts them (to stay within OV.budget), the
* second makes them again and keeps them, then they are filed by shape in typed arrays. overlapIndexStep() does it a slice at a time (ms per slice; the page builds it between other work);
* overlapIndex() all at once.
*/
function overlapIndexStep(db, ms) {
if (db.ov) return true;
const b = db.ovBuild || (db.ovBuild = { pass: 1, i: 0, entries: [], hist: new Float64Array(256) }), t0 = Date.now();
if (b.pass === 1) {
while (b.i < db.seeds.length) {
const i = b.i++, s = db.seeds[i];
if (!db.extOK[i] || (s.isPart && !s.isPiece) || (s.isPiece && !OV_PIECES.includes(s.pieceKind))) continue;
const det = detailBytes(s.detail), asp = det && entryAspect(s);
if (!det || !(asp > 0.34 && asp < 3)) continue;
const n = entryPatterns(b, det, asp), sds = b.sds, hist = b.hist;
for (let t = 0; t < n; t++) hist[sds[t]]++;
b.entries.push({ i, det, aspect: asp });
if (Date.now() - t0 > ms) return false;
}
let cut = 0, total = b.hist.reduce((x, y) => x + y, 0);
for (; total > OV.budget && cut < 255; cut++) total -= b.hist[cut];
Object.assign(b, { pass: 2, e: 0, cut, n: 0, allKeys: new Uint16Array(total), allVals: new Uint32Array(total) });
}
while (b.e < b.entries.length) {
const e = b.e++, E = b.entries[e], n = entryPatterns(b, E.det, E.aspect), cut = b.cut;
for (let t = 0; t < n; t++) {
if (b.sds[t] < cut) continue;
b.allKeys[b.n] = b.keys[t];
b.allVals[b.n++] = (e << 13 | b.offs[t]) >>> 0; // the reference, then the pattern's place
}
if (Date.now() - t0 > ms) return false;
}
// filed by shape (in the order made: by reference, then place)
const start = new Uint32Array(65537), keys = b.allKeys, vals = b.allVals;
for (let t = 0; t < b.n; t++) start[keys[t] + 1]++;
for (let k = 1; k <= 65536; k++) start[k] += start[k - 1];
const at = start.slice(0, 65536), pk = new Uint32Array(start[65536]);
for (let t = 0; t < b.n; t++) pk[at[keys[t]]++] = vals[t];
db.ov = { entries: b.entries, start, pk, boards: new Map() };
delete db.ovBuild;
return true;
}
function overlapIndex(db) {
overlapIndexStep(db, Infinity);
return db.ov;
}
/** Whether matchOverlap() can run now: always, unless the page is still building the index in the background. */
const overlapReady = db => !!db.ov || !db.ovBackground;
/** Correlation of a reference's detail with the picture's cells at a placement, over the cells mostly inside it. */
function overlapCorr(T, cx, cy, W, H, det) {
gridCells(T, cx - W / 2, cy - H / 2, cx + W / 2, cy + H / 2, DETAIL, DETAIL, DM, DV);
let n = 0, sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0;
for (let k = 0; k < DETAIL * DETAIL; k++) {
if (DV[k] < EXT_VISIBLE) continue;
const x = DM[k], y = det[k];
n++; sx += x; sy += y; sxx += x * x; syy += y * y; sxy += x * y;
}
if (n < OV.cells) return { corr: -1, n };
const vx = sxx - sx * sx / n, vy = syy - sy * sy / n;
return vx > 16 * n && vy > 16 * n ? { corr: (sxy - sx * sy / n) / Math.sqrt(vx * vy), n } : { corr: -1, n };
}
/**
* Whether the detail agrees in each quarter of the reference (2x2 blocks of 6x6 cells) that the placement
* covers, when it covers fewer than three.
*/
function overlapQuarters(T, cx, cy, W, H, det) {
gridCells(T, cx - W / 2, cy - H / 2, cx + W / 2, cy + H / 2, DETAIL, DETAIL, DM, DV);
const q = [];
for (let qy = 0; qy < 2; qy++) for (let qx = 0; qx < 2; qx++) {
let n = 0, sx = 0, sy = 0, sxx = 0, syy = 0, sxy = 0;
for (let y = qy * 6; y < qy * 6 + 6; y++) for (let x = qx * 6; x < qx * 6 + 6; x++) {
const k = y * DETAIL + x;
if (DV[k] < EXT_VISIBLE) continue;
const u = DM[k], v = det[k];
n++; sx += u; sy += v; sxx += u * u; syy += v * v; sxy += u * v;
}
const vx = sxx - sx * sx / n, vy = syy - sy * sy / n;
if (n >= 12 && vx > 16 * n && vy > 16 * n) q.push((sxy - sx * sy / n) / Math.sqrt(vx * vy));
}
return q.length >= 3 || q.every(c => c >= OV.quarter);
}
/** Best placement near (cx, cy, W) by the detail's correlation: coordinate descent over centre and width. */
function overlapTune(T, f, det, aspect) {
let cur = { ...f, ...overlapCorr(T, f.cx, f.cy, f.W, f.W * aspect, det) };
for (const st of [0.5, 0.25, 0.125]) {
for (let it = 0, moved = true; moved && it < 8; it++) {
moved = false;
const dx = st * cur.W / DETAIL;
for (const [mx, my, mw] of [[1, 0, 0], [-1, 0, 0], [0, 1, 0], [0, -1, 0], [0, 0, 1], [0, 0, -1]]) {
const W = cur.W * (1 + mw * st * 0.08), cx = cur.cx + mx * dx, cy = cur.cy + my * dx;
const r = overlapCorr(T, cx, cy, W, W * aspect, det);
if (r.corr > cur.corr + 1e-6) { cur = { ...cur, cx, cy, W, ...r }; moved = true; }
}
}
}
return cur;
}
/**
* Overlapping crops of a picture (grayCopy) with no other match, against the references that apply
* on `board` and have a detail and a known shape. Returns {seed, how: 'overlap', a64, d64, corr,
* cells, size, tilt, rule} for the best placement that passes every check, or null. stats (optional)
* counts patterns and checked placements; stats.fits (calibration) records every checked placement.
*/
/** A count per whole-number key (below 2^53), kept in typed arrays and reused: `slots` in the order first counted. */
function countTable(bits) { return { bits, k: new Float64Array(1 << bits), v: new Uint32Array(1 << bits), slots: [] }; }
function countSlot(t, k) {
const lo = k % 4294967296, hi = (k - lo) / 4294967296, mask = (1 << t.bits) - 1;
let s = Math.imul((lo | 0) ^ Math.imul(hi | 0, 0x9e3779b1), 0x85ebca6b) >>> (32 - t.bits);
while (t.v[s] && t.k[s] !== k) s = (s + 1) & mask;
return s;
}
function countAdd(t, k) {
if (2 * t.slots.length >= t.v.length) { // half full: twice the size, the same order
const old = t.slots.map(s => [t.k[s], t.v[s]]);
t.bits++; t.k = new Float64Array(1 << t.bits); t.v = new Uint32Array(1 << t.bits); t.slots = [];
for (const [k, v] of old) { const s = countSlot(t, k); t.k[s] = k; t.v[s] = v; t.slots.push(s); }
}
const s = countSlot(t, k);
if (t.v[s]++ === 0) { t.k[s] = k; t.slots.push(s); }
}
function countClear(t) { for (const s of t.slots) t.v[s] = 0; t.slots.length = 0; }
function matchOverlap(px, db, cfg, board, stats) {
if (px.w < 24 || px.h < 24 || !overlapReady(db)) return null;
const ix = overlapIndex(db);
if (!ix.entries.length) return null;
if (!ix.boards.has(board)) { const ok = new Uint8Array(ix.entries.length), on = new Set(boardIndex(db, board)); ix.entries.forEach((e, k) => { ok[k] = on.has(e.i) ? 1 : 0; }); ix.boards.set(board, ok); }
const okE = ix.boards.get(board), T = integral(px), weak = [0, 0];
// The picture's patterns at every size, once: [lattice index, probes...] per size.
const pats = [];
let patterns = 0;
for (let si = 0; si < OV.steps; si++) {
const c = OV.sizes[0] * Math.pow(OV.sizes[1] / OV.sizes[0], si / (OV.steps - 1)) * px.w / DETAIL; // the reference's cell, in pixels
const nx = Math.min(64, Math.floor(px.w / c)), ny = Math.min(64, Math.floor(px.h / c));
if (nx < 4 || ny < 4) continue;
const lat = new Float64Array(nx * ny);
gridLattice(T, 0, 0, c, c, nx, ny, lat);
for (let j = 0; j <= ny - 4; j++) for (let i = 0; i <= nx - 4; i++) {
const key = patternKey(lat, nx, i, j, weak, OV.minStdIn);
if (key < 0) continue;
patterns++;
for (const probe of [key, key ^ (1 << weak[0])]) { // and with its least certain cell the other way
if (ix.start[probe + 1] - ix.start[probe] <= OV.common) pats.push(si, i, j, probe);
}
}
}
if (stats) stats.patterns = (stats.patterns || 0) + patterns;
// First, votes per reference, size and side only (a plain count); then, for the few with most, where.
const nE = ix.entries.length;
if (!ix.count || ix.count.length < nE * 32) ix.count = new Uint16Array(nE * 32);
const count = ix.count, touched = [];
for (let t = 0; t < pats.length; t += 4) {
const si = pats[t], probe = pats[t + 3], end = ix.start[probe + 1], w = OV_RARE[end - ix.start[probe]];
for (let p = ix.start[probe]; p < end; p++) {
const e = ix.pk[p] >>> 13;
if (!okE[e]) continue;
const k = e * 32 + si + 16 * ((ix.pk[p] >> 12) & 1), c = count[k];
if (c === 0) touched.push(k);
count[k] = c + w;
}
}
// the most voted, ties to the lower slot (the order a full scan would give)
const lead = [], before = (x, y) => x[1] > y[1] || (x[1] === y[1] && x[0] < y[0]);
for (const k of touched) {
const v = count[k], last = lead.length === OV.lead && lead[lead.length - 1];
if (v < OV.votes || (last && (v < last[1] || (v === last[1] && k > last[0])))) continue;
const c = [k, v];
let q = lead.length;
while (q > 0 && before(c, lead[q - 1])) q--;
lead.splice(q, 0, c);
if (lead.length > OV.lead) lead.pop();
}
for (const k of touched) count[k] = 0; // left clean for the next picture
if (!ix.chosen || ix.chosen.length < nE * 32) ix.chosen = new Uint8Array(nE * 32);
const chosen = ix.chosen;
for (const [k] of lead) chosen[k] = 1;
if (!ix.votes) ix.votes = countTable(14);
const votes = ix.votes;
for (let t = 0; lead.length && t < pats.length; t += 4) {
const si = pats[t], i = pats[t + 1], j = pats[t + 2], probe = pats[t + 3];
for (let p = ix.start[probe], end = ix.start[probe + 1]; p < end; p++) {
const e = ix.pk[p] >>> 13, o = ix.pk[p] & 8191, sm = si + 16 * (o >> 12);
if (!chosen[e * 32 + sm]) continue;
const qx = 2 * i - (o & 63), qy = 2 * j - ((o >> 6) & 63); // the reference's corner, in half cells
const k = ((e * 32 + sm) * 512 + qx + 256) * 512 + qy + 256;
countAdd(votes, k);
}
}
for (const [k] of lead) chosen[k] = 0;
// A placement's votes are pooled with its neighbours' (half a cell either way: a crop rarely lands on the
// grid), and only the peak of each cluster is kept.
const top = [];
for (const s of votes.slots) {
const k = votes.k[s], own = votes.v[s];
let v = 0, peak = true;
for (let dx = -512; dx <= 512 && peak; dx += 512) for (let dy = -1; dy <= 1; dy++) {
const n = votes.v[countSlot(votes, k + dx + dy)];
if (!n) continue;
v += n;
if (n > own || (n === own && k + dx + dy < k)) { peak = false; break; }
}
if (!peak || v < OV.votes || (top.length === OV.shortlist && v <= top[top.length - 1][1])) continue;
let q = top.length;
while (q > 0 && top[q - 1][1] < v) q--;
top.splice(q, 0, [k, v]);
if (top.length > OV.shortlist) top.pop();
}
countClear(votes); // left empty for the next picture
// A mirrored reference is compared with the mirrored picture, against the reference as stored.
const views = [{ px, T, turned: new Map() }, null];
const view = m => {
if (m && !views[1]) {
const g = new Uint8ClampedArray(px.g.length);
for (let y = 0; y < px.h; y++) for (let x = 0; x < px.w; x++) g[y * px.w + x] = px.g[y * px.w + px.w - 1 - x];
const mp = { g, w: px.w, h: px.h };
views[1] = { px: mp, T: integral(mp), turned: new Map() };
}
return views[m];
};
// Each shortlisted placement's detail compared where it was voted (cheap); the best few are tuned.
const cands = [];
for (const [k, v] of top) {
const qy = k % 512 - 256, r1 = (k - qy - 256) / 512, qx = r1 % 512 - 256, r2 = (r1 - qx - 256) / 512, sm = r2 % 32, e = (r2 - sm) / 32;
const si = sm % 16, mirror = sm >> 4, E = ix.entries[e];
const c = OV.sizes[0] * Math.pow(OV.sizes[1] / OV.sizes[0], si / (OV.steps - 1)) * px.w / DETAIL, W0 = c * DETAIL;
const cx0 = qx / 2 * c + W0 / 2, at = { cx: mirror ? px.w - cx0 : cx0, cy: qy / 2 * c + W0 * E.aspect / 2, W: W0 };
const c0 = overlapCorr(view(mirror).T, at.cx, at.cy, at.W, at.W * E.aspect, E.det).corr;
if (c0 >= OV.start) cands.push({ E, mirror, at, v, c0 });
}
cands.sort((x, y) => y.c0 - x.c0);
let best = null;
for (const { E, mirror, at, v } of cands.slice(0, OV.verify)) {
const seed = db.seeds[E.i], vw = view(mirror), VT = vw.T, turned = vw.turned;
// straight, then turned back by small angles (the placement carried over to the turned picture),
// only when the straight one is promising but not yet enough
let f = { ...overlapTune(VT, at, E.det, E.aspect), tilt: 0, T: VT };
for (const deg of f.corr >= OV.turn && f.corr < OV.corr ? OV.tilts : []) {
if (!turned.has(deg)) { const sub = turnedPicture(vw.px, deg, 1); turned.set(deg, sub && { T: integral(sub), w: sub.w, h: sub.h }); }
const t = turned.get(deg);
if (!t) continue;
const a = deg * Math.PI / 180, cs = Math.cos(a), sn = Math.sin(a), dx = f.cx - px.w / 2, dy = f.cy - px.h / 2;
const g = overlapTune(t.T, { cx: dx * cs + dy * sn + t.w / 2, cy: -dx * sn + dy * cs + t.h / 2, W: f.W }, E.det, E.aspect);
if (g.corr > f.corr) f = { ...g, tilt: deg, T: t.T };
}
if (stats) stats.checked = (stats.checked || 0) + 1;
if (f.corr < 0) continue;
const H = f.W * E.aspect, h = gridDistance(f.T, f.cx - f.W / 2, f.cy - H / 2, f.cx + f.W / 2, f.cy + H / 2, db.words, E.i * 4, {});
const r = db.ruleIdx[E.i] ? db.rules[db.ruleIdx[E.i]] : cfg, size = f.W / px.w;
const fit = { seed, how: 'overlap', a64: h.a64, d64: h.d64, info: h.info, na: h.na, nd: h.nd, corr: f.corr, cells: f.n, size, tilt: f.tilt, mirror, votes: v, rule: r };
if (stats && stats.fits) stats.fits.push({ i: E.i, how: 'overlap', a64: h.a64, d64: h.d64, info: h.info, na: h.na, nd: h.nd, corr: f.corr, cells: f.n, size, tilt: f.tilt, mirror, votes: v });
// the reference's own limits, if stricter than the general defaults, make these stricter too
const A = OV.a * Math.min(1, r.a / 4), D = OV.d * Math.min(1, r.d / 5);
if (!r.d || h.na < OV.bits || h.nd < OV.bits || h.a64 > A || h.d64 > D || h.info < (f.tilt ? OV.infoTurned : OV.info) || f.corr < OV.corr
|| size < OV.sizes[0] || size > OV.sizes[1] || !overlapQuarters(f.T, f.cx, f.cy, f.W, H, E.det)) continue;
if (!best || f.corr > best.corr) best = fit;
}
return best;
}
/** Cell means of an nx x ny lattice of cw x ch cells from (ox, oy), into `out` (row by row). */
function gridLattice(T, ox, oy, cw, ch, nx, ny, out) {
const { S, w, h } = T, W = w + 1, X = new Float64Array(nx + 1), Y = new Float64Array(ny + 1), P = new Float64Array((nx + 1) * (ny + 1));
for (let i = 0; i <= nx; i++) X[i] = Math.min(w, ox + i * cw);
for (let j = 0; j <= ny; j++) Y[j] = Math.min(h, oy + j * ch);
for (let j = 0; j <= ny; j++) {
const y = Y[j], yi = Math.min(h - 1, Math.floor(y)), fy = y - yi;
for (let i = 0; i <= nx; i++) {
const x = X[i], xi = Math.min(w - 1, Math.floor(x)), fx = x - xi, o = yi * W + xi;
const a = S[o], b = S[o + 1], c = S[o + W], d = S[o + W + 1];
P[j * (nx + 1) + i] = a + (b - a) * fx + (c - a) * fy + (a - b - c + d) * fx * fy;
}
}
const area = cw * ch;
for (let j = 0; j < ny; j++) for (let i = 0; i < nx; i++) {
const p = j * (nx + 1) + i;
out[j * nx + i] = (P[p + nx + 2] - P[p + 1] - P[p + nx + 1] + P[p]) / area;
}
}
// Archive table: hosts, FoolFuuka flags (else Fuuka-style URLs), and per board every archive
// that has it as [host index, flags (1 = keeps files)], see buildArchiveTable().
// BUNDLED_ARCHIVES is buildArchiveTable() of https://4chenz.github.io/archives.json/archives.json
// (the list 4chan XT and 4plebs use) as of 2026-10-28.
const BUNDLED_ARCHIVES = {
v: 2,
hosts: ['archive.4plebs.org', 'warosu.org', 'desuarchive.org', 'boards.fireden.net', 'arch.b4k.dev', 'archived.moe', 'thebarchive.com', 'archiveofsins.com', 'archive.palanq.win', 'eientei.xyz'],
ff: [1, 0, 1, 1, 1, 1, 1, 1, 1, 0],
boards: {
'3':[[1,1],[9,1],[5,0]],a:[[2,1],[5,0]],aco:[[2,1],[5,0]],adv:[[0,1],[5,0]],an:[[2,1],[5,0]],
asp:[[5,0]],b:[[6,1],[5,0]],bant:[[6,1],[8,1],[5,0]],biz:[[1,1],[5,0]],c:[[2,1],[8,1],[5,0]],
can:[[5,1]],cgl:[[1,1],[2,1],[5,0]],ck:[[1,1],[5,0]],cm:[[3,1],[5,0]],co:[[2,1],[3,1],[5,0]],
cock:[[5,1]],con:[[5,1],[8,0]],d:[[2,1],[5,0]],diy:[[1,1],[5,0]],e:[[8,1],[5,0]],f:[[0,1],[5,0]],
fa:[[1,1],[5,0]],fap:[[5,1]],fit:[[2,1],[5,0]],fitlit:[[5,1]],g:[[2,1],[4,0],[5,0]],gd:[[5,1]],
gif:[[5,0]],h:[[7,1],[5,0]],hc:[[7,1],[5,0]],his:[[2,1],[5,0]],hm:[[7,1],[5,0]],hr:[[0,1],[5,0]],
i:[[7,1],[8,1],[9,1],[5,0]],ic:[[1,1],[3,1],[5,0]],int:[[2,1],[5,0]],jp:[[1,1],[5,0]],
k:[[2,1],[5,0]],lgbt:[[7,1],[5,0]],lit:[[1,1],[5,0]],m:[[2,1],[5,0]],mlp:[[2,1],[4,0],[5,0]],
mlpol:[[0,1],[5,1]],mo:[[0,1],[5,1]],mtv:[[5,1]],mu:[[2,1],[5,0]],n:[[8,1],[5,0]],
news:[[8,1],[5,0]],o:[[0,1],[5,0]],out:[[8,1],[5,0]],outsoc:[[5,1]],p:[[8,1],[5,0]],po:[[5,1]],
pol:[[0,1],[5,0]],pw:[[8,1],[5,0]],q:[[2,1],[5,1]],qa:[[2,1],[5,0]],qb:[[4,1],[5,1]],
qst:[[5,1],[8,1]],r:[[7,1],[5,0]],r9k:[[2,1],[5,0]],s:[[7,1],[5,0]],s4s:[[0,1],[5,0]],
sci:[[1,1],[3,1],[9,1],[5,0]],soc:[[7,1],[5,0]],sp:[[0,1],[5,0]],spa:[[5,1]],t:[[7,1],[5,0]],
tg:[[0,1],[2,1],[5,0]],toy:[[8,1],[5,0]],trash:[[2,1],[5,0]],trv:[[0,1],[5,0]],tv:[[0,1],[5,0]],
u:[[7,1],[5,0]],v:[[4,1],[5,0]],vg:[[4,1],[5,0]],vint:[[5,1]],vip:[[3,1],[5,1],[8,1]],
vm:[[4,1],[5,0]],vmg:[[4,1],[5,0]],vp:[[4,1],[8,1],[5,0]],vr:[[1,1],[2,1],[5,0]],vrpg:[[4,1],[5,0]],
vst:[[4,1],[5,0]],vt:[[1,1],[8,1],[5,0]],w:[[8,1],[5,0]],wg:[[8,1],[5,0]],wsg:[[2,0],[5,0]],
wsr:[[8,1],[5,0]],x:[[0,1],[5,0]],xs:[[9,1],[5,0]],y:[[3,1],[5,0]]
},
};
/**
* archives.json -> compact table: hosts, FoolFuuka flags, and per board every archive that has it
* as [hostIndex, flags] (1 = keeps files), file-keeping archives first. The list's "search" field
* is not used: it is missing for most archives that do support image search (Desuarchive, 4plebs,
* ...), and 4chan XT ignores it too.
* Entries with anything but a plain hostname, or without HTTPS, are dropped, so a bad list cannot
* produce odd links.
*/
function buildArchiveTable(list) {
const HOST = /^[a-z0-9-]+(\.[a-z0-9-]+)+$/i, BOARD = /^[a-z0-9]{1,12}$/;
const field = (a, k) => (Array.isArray(a[k]) ? a[k] : []);
const arch = (Array.isArray(list) ? list : []).filter(a => a && typeof a.domain === 'string' && HOST.test(a.domain) && a.https !== false);
const boards = {};
for (const b of new Set(arch.flatMap(a => field(a, 'boards')))) {
if (typeof b !== 'string' || !BOARD.test(b)) continue;
const cands = arch.map((a, i) => [i, field(a, 'files').includes(b) ? 1 : 0])
.filter(([i]) => field(arch[i], 'boards').includes(b));
boards[b] = cands.sort((x, y) => (y[1] & 1) - (x[1] & 1));
}
return { v: 2, hosts: arch.map(a => a.domain.toLowerCase()), ff: arch.map(a => (a.software === 'foolfuuka' ? 1 : 0)), boards };
}
/**
* Archives used for a board: { post, search } host indices. `pick` is the user's choice for that
* board, { post: host, search: host }, as in 4chan XT's per-board archive settings. Like XT, image
* search defaults to the archive used for viewing. A choice that is no longer in the list falls
* back to the automatic one.
*/
function pickArchives(table, board, pick) {
const cands = Object.prototype.hasOwnProperty.call(table.boards, board) && table.boards[board];
if (!cands || !cands.length) return null;
const chosen = (host, fallback) => (host && cands.find(([i]) => table.hosts[i] === host)) || fallback;
const post = chosen(pick && pick.post, cands[0]);
return { post: post[0], search: chosen(pick && pick.search, post)[0] };
}
/**
* Plain archive links for a reference. Nothing is fetched: the browser only goes there if clicked.
* Needs the source board; `post` is null without a post number, `search` null without an MD5.
*/
function archiveLinks(seed, table, pick) {
const use = pickArchives(table, seed.board, pick);
const hasPost = /^\d+$/.test(String(seed.post || ''));
if (!use || (!hasPost && !seed.md5)) return null;
const { post: p, search: s } = use, ff = table.ff[p], base = i => `https://${table.hosts[i]}/${seed.board}/`;
const thread = /^\d+$/.test(String(seed.thread || '')) ? seed.thread : null;
const post = !hasPost ? null : base(p) + (thread
? `thread/${thread}${ff ? '/' : ''}#${ff ? '' : 'p'}${seed.post}`
: `post/${seed.post}${ff ? '/' : ''}`);
let search = null;
if (seed.md5) {
const md5 = encodeURIComponent(seed.md5.replace(/[+/=]/g, c => ({ '+': '-', '/': '_', '=': '' })[c]));
search = base(s) + (table.ff[s] ? `search/image/${md5}/` : `image/${md5}`);
}
return { host: table.hosts[p], post, searchHost: table.hosts[s], search };
}
// ---- Looking a file up in an archive (only when you start it: see lookupInArchive())
// Archives that serve thumbnails from another host than their own. A thumbnail link anywhere else is refused.
const ARCHIVE_IMAGE_HOSTS = { 'archive.4plebs.org': ['i.4pcdn.org'], 'desuarchive.org': ['desu-usergeneratedcontent.xyz'],
'boards.fireden.net': ['img-lb.fireden.net'], 'arch.b4k.dev': ['arch-img.b4k.dev'], 'archive.palanq.win': ['archive-media.palanq.win'] };
/** FoolFuuka image-search API address for a file's MD5 on one board of archive `i`, or null. */
function archiveSearchUrl(table, i, board, md5) {
const m = normMd5(md5);
if (!table.ff[i] || !table.hosts[i] || !/^[a-z0-9]{1,12}$/.test(board) || !m) return null;
return `https://${table.hosts[i]}/_/api/chan/search/?boards=${board}&image=${m.replace(/[+/=]/g, c => ({ '+': '-', '/': '_', '=': '' })[c])}`;
}
/** A thumbnail address on that archive's own hosts: plain HTTPS to an image file, nothing else. */
function archiveThumbOk(url, host) {
if (typeof url !== 'string' || /\.\.|%|\\|\s/.test(url)) return false; // nothing a URL parser would rewrite
let u;
try { u = new URL(url); } catch (e) { return false; }
return u.protocol === 'https:' && !u.username && !u.password && !u.port && !u.search && !u.hash &&
[host, ...(ARCHIVE_IMAGE_HOSTS[host] || [])].includes(u.hostname) &&
/^\/[\w./-]{1,300}\.(jpe?g|png|gif|webp)$/i.test(u.pathname) && !u.pathname.includes('..');
}
/** In an archive's search answer, the first post whose file has exactly this MD5 and a usable thumbnail. */
function archiveFind(answer, md5, host) {
const want = normMd5(md5);
for (const group of answer && typeof answer === 'object' ? Object.values(answer) : []) {
for (const p of group && Array.isArray(group.posts) ? group.posts : []) {
const m = p && p.media;
if (want && m && normMd5(m.media_hash) === want && archiveThumbOk(m.thumb_link, host)) return { thumb: m.thumb_link };
}
}
return null;
}
// Exported lists are sorted by hash (MD5-only entries after picture ones, by MD5).
const hashKey = x => (x.a ? x.a + x.d : `~${x.md5}`);
const byHash = (x, y) => (hashKey(x) < hashKey(y) ? -1 : hashKey(x) > hashKey(y) ? 1 : 0);
const isPicture = s => !!s && s.algo === HASH_ALGO && HEX64.test(s.a) && HEX64.test(s.d);
const DATE = /^\d{4}-\d{2}-\d{2}/;
/**
* Shareable blacklist, sorted by hash so the order says nothing about when entries were added.
* Always: hash method + the two hashes (all matching needs), or for an exact-file (MD5-only)
* reference its MD5. Optional (opts):
* source - image MD5 and the board it was blacklisted on (together: an MD5 search needs a board)
* post - the exact post: post and thread number (only with source, since it needs the board)
* boards - the boards each reference applies to, its own or its category's ("effective"), so a
* reference restricted through its category stays restricted in versions without categories
* dates - the date each reference was added
* cats - categories (name, enabled, boards, rules), listed once, and each reference's tags
* With everything on, an imported reference looks as if the importer had added it themselves.
* Never exported: local IDs, image size, thumbnail URL, script version.
*/
/** A category's settings as exported: the 4chan X-family ones belong to this browser and stay here. */
const LOCAL_SETTINGS = ['xAdd', 'xAuto', 'xNoStub', 'workers'];
function portableSettings(set) {
const out = {};
for (const [k, v] of Object.entries(set || {})) if (!LOCAL_SETTINGS.includes(k)) out[k] = v;
return Object.keys(out).length ? { settings: out } : {};
}
function blacklistFile(seeds, opts = {}, cats = []) {
const catById = new Map(cats.map(c => [c.id, c])), catOut = [], catIndex = new Map();
const refs = seeds.filter(s => isPicture(s) || (s && normMd5(s.md5)))
.map(s => {
const md5 = normMd5(s.md5), r = isPicture(s) ? { algo: s.algo, a: s.a, d: s.d } : { md5 }, c = catById.get(s.cat);
if (DETAIL_TEXT.test(s.detail || '')) r.detail = s.detail; // matching data, not metadata: always exported
if (r.a && s.blank === true) r.blank = true; // likewise (see matchPrepared)
if (r.a && s.part && HEX64.test(s.part.a)) r.part = { ...s.part }; // and its main picture
const pt = r.a && opts.pieces !== false && piecesText(s.pieces); // and its pieces, unless left out
if (pt) r.pieces = pt;
if (opts.source && s.board && md5) { r.md5 = md5; r.board = s.board; }
if (opts.source && opts.post && s.board && /^\d+$/.test(String(s.post || ''))) {
r.board = s.board;
r.post = String(s.post);
if (/^\d+$/.test(String(s.thread || ''))) r.thread = String(s.thread);
}
const scope = Array.isArray(s.boards) && s.boards.length ? s.boards : c && Array.isArray(c.boards) && c.boards.length ? c.boards : null;
if (opts.boards && scope) r.boards = scope.slice();
if (opts.dates && DATE.test(s.added)) r.added = s.added.slice(0, 10);
if (opts.cats) {
if (c) {
if (!catIndex.has(c.id)) {
catIndex.set(c.id, catOut.length);
catOut.push({ name: c.name, ...(c.enabled === false ? { enabled: false } : {}),
...(Array.isArray(c.boards) && c.boards.length ? { boards: c.boards.slice() } : {}), ...(c.rules ? { rules: { ...c.rules } } : {}),
...portableSettings(c.settings) });
}
r.cat = catIndex.get(c.id);
}
if (Array.isArray(s.tags) && s.tags.length) r.tags = s.tags.slice();
}
return r;
})
.sort(byHash);
return catOut.length ? { format: 'better-filter-references', categories: catOut, references: refs }
: { format: 'better-filter-references', references: refs };
}
/**
* Shareable false-positive list, sorted by hash. Always: the image's two hashes (all the whitelist
* needs), or the MD5 of an exact-file entry that has no hashes. Optional (opts):
* post - where it happened: board, post and thread number, and the file's MD5
* match - the hashes (or MD5) of the reference it had matched, and the two distances
* dates - the date each entry was added
* `seeds` is the exporter's reference list, used to turn local reference IDs into hashes.
*/
function falsePositivesFile(allow, seeds, opts = {}) {
const items = allow.filter(x => x && ((HEX64.test(x.a) && HEX64.test(x.d)) || normMd5(x.md5)))
.map(x => {
const r = HEX64.test(x.a) && HEX64.test(x.d) ? { a: x.a, d: x.d } : { md5: normMd5(x.md5) };
if (opts.post && normMd5(x.md5)) r.md5 = normMd5(x.md5);
if (opts.post && /^[a-z0-9]{1,12}\.\d+$/.test(x.post || '')) {
r.post = x.post;
if (/^\d+$/.test(String(x.thread || ''))) r.thread = String(x.thread);
}
if (opts.match) {
const ref = seeds.find(y => y.id === x.seed);
if (isPicture(ref)) r.matched = { algo: ref.algo, a: ref.a, d: ref.d };
else if (ref && normMd5(ref.md5)) r.matched = { md5: normMd5(ref.md5) };
if (Number.isInteger(x.da) && Number.isInteger(x.dd)) { r.da = x.da; r.dd = x.dd; }
}
if (opts.dates && DATE.test(x.added)) r.added = x.added.slice(0, 10);
return r;
})
.sort(byHash);
return { format: 'better-filter-false-positives', falsePositives: items };
}
// ---- 4chan X-family MD5 filter rules (see "4chan X family" below for how they are added)
/** One rule line for an MD5, with the reference's boards (never broader than it applies) and stub choice. */
function xRuleLine(md5, boards, stub) {
return `/${md5}/` + (boards && boards.length ? `;boards:${boards.join(',')}` : '') + (stub === false ? ';stub:no' : '') + (boards && boards.length || stub === false ? ';' : '');
}
// "/<MD5>/flags;options". An MD5 can contain "/", so the pattern ends at the "/" before the options.
const X_RULE = /^\/(.+?)\/(\w*)(;.*)?$/;
/** The MD5 of one rule line, also of a commented-out one ("#/.../"); null if it isn't an MD5 rule. */
function xLineMd5(line) {
const m = X_RULE.exec(String(line).trim().replace(/^#\s*/, ''));
return m ? normMd5(m[1]) : null;
}
/**
* Parse an X-family MD5 filter (their "Image MD5" textarea). Lines Better Filter can't represent
* without changing what they filter are reported, not converted: commented-out rules, highlight-
* or notify-only rules, exclude:, op:, file:, and sfw/nsfw or other sites' board lists.
* Returns { rules: [{md5, boards, line}], skipped: {reason: count}, invalid }.
*/
function parseXFilter(text) {
const out = { rules: [], skipped: {}, invalid: 0 };
const skip = why => { out.skipped[why] = (out.skipped[why] || 0) + 1; };
for (const raw of String(text).split(/\r?\n/)) {
const line = raw.trim();
if (!line) continue;
if (line[0] === '#') { skip('commented out'); continue; }
const m = X_RULE.exec(line), md5 = m && normMd5(m[1]);
if (!md5) { out.invalid++; continue; }
const opts = m[3] || '', opt = k => (new RegExp(`(?:^|;)\\s*${k}:([^;]*)`).exec(opts) || [])[1];
if (/(?:^|;)\s*(highlight|notify)/.test(opts) && !/(?:^|;)\s*hide(?:[;:]|$)/.test(opts)) { skip('highlight or notify only'); continue; }
if (opt('exclude') || opt('op') || opt('file')) { skip('exclude:, op: or file: option'); continue; }
let boards = null;
if (opt('boards') !== undefined) {
boards = [];
for (const b of opt('boards').split(',').map(x => x.trim().toLowerCase()).filter(Boolean)) {
const [site, id] = b.includes(':') ? b.split(':').slice(-2) : ['', b];
if (site && !/^(4chan\.org|4channel\.org|boards\.4chan(nel)?\.org)$/.test(site)) { boards = null; break; }
if (!/^[a-z0-9]{1,12}$/.test(id) || id === 'sfw' || id === 'nsfw') { boards = null; break; }
boards.push(id);
}
if (!boards || !boards.length) { skip('board list Better Filter can\'t represent'); continue; }
}
out.rules.push({ md5, boards, line });
}
return out;
}
// #endregion MBGX-CORE
return {
HASH_ALGO, toGray, axisWeights, boxResize, bitsToHex, hashPixels, hashGray, LOW_STEPS, WEAK,
thumbnailWeakness, cellInfo, HEX64, popcount32, hamming, prepareReferences, normMd5, matchMd5, boardIndex,
prepareAllow, matchPrepared, evaluate, grayCopy, integral, GX, GXI, M8, gridCells, windowWords,
EXT_VISIBLE, EXT_MIN_BITS, EXT_FLAT, bitAt, kthSmallest, gridDistance, MOVES, alignReference, DETAIL,
detailOf, DETAIL_TEXT, detailBytes, detailCorrelation, ORD_SUM, ordinaryDetail, LOOK, matchLookalike,
contentBox, blockBox, pictureOf, PIECES, piecesOf, PIECE_BYTES, packPieces, unpackPieces, piecesText,
pieceCount, panelBoxes, ZOOM_KEEP, quarterTurn, innerPicture, turnedPicture, EXT_RULES, EXT_PICK,
EXT_PER_WINDOW, EXT_START, EXT_START_FOR, matchExtended, OV, OV_PIECES, OV_RARE, entryAspect, PATTERN_SD,
patternKey, squareGrid, overlapIndexStep, overlapIndex, overlapReady, overlapCorr, overlapQuarters,
overlapTune, countTable, countSlot, countAdd, countClear, matchOverlap, gridLattice, BUNDLED_ARCHIVES,
buildArchiveTable, pickArchives, archiveLinks, ARCHIVE_IMAGE_HOSTS, archiveSearchUrl, archiveThumbOk,
archiveFind, hashKey, byHash, isPicture, DATE, LOCAL_SETTINGS, portableSettings, blacklistFile,
falsePositivesFile, xRuleLine, X_RULE, xLineMd5, parseXFilter, LOW_RESID, GY, GP, GXF, V8, M9, V9, RANK,
DM, DV, ORD_DETAIL, CLOSE, PIECE_TEXT, TILT_DEGREES, TILT_KEEP, EXT_DETAIL
};
}
const {
HASH_ALGO, toGray, axisWeights, boxResize, bitsToHex, hashPixels, hashGray, LOW_STEPS, WEAK,
thumbnailWeakness, cellInfo, HEX64, popcount32, hamming, prepareReferences, normMd5, matchMd5, boardIndex,
prepareAllow, matchPrepared, evaluate, grayCopy, integral, GX, GXI, M8, gridCells, windowWords,
EXT_VISIBLE, EXT_MIN_BITS, EXT_FLAT, bitAt, kthSmallest, gridDistance, MOVES, alignReference, DETAIL,
detailOf, DETAIL_TEXT, detailBytes, detailCorrelation, ORD_SUM, ordinaryDetail, LOOK, matchLookalike,
contentBox, blockBox, pictureOf, PIECES, piecesOf, PIECE_BYTES, packPieces, unpackPieces, piecesText,
pieceCount, panelBoxes, ZOOM_KEEP, quarterTurn, innerPicture, turnedPicture, EXT_RULES, EXT_PICK,
EXT_PER_WINDOW, EXT_START, EXT_START_FOR, matchExtended, OV, OV_PIECES, OV_RARE, entryAspect, PATTERN_SD,
patternKey, squareGrid, overlapIndexStep, overlapIndex, overlapReady, overlapCorr, overlapQuarters,
overlapTune, countTable, countSlot, countAdd, countClear, matchOverlap, gridLattice, BUNDLED_ARCHIVES,
buildArchiveTable, pickArchives, archiveLinks, ARCHIVE_IMAGE_HOSTS, archiveSearchUrl, archiveThumbOk,
archiveFind, hashKey, byHash, isPicture, DATE, LOCAL_SETTINGS, portableSettings, blacklistFile,
falsePositivesFile, xRuleLine, X_RULE, xLineMd5, parseXFilter, LOW_RESID, GY, GP, GXF, V8, M9, V9, RANK,
DM, DV, ORD_DETAIL, CLOSE, PIECE_TEXT, TILT_DEGREES, TILT_KEEP, EXT_DETAIL
} = mbgxCore();
// ---------------------------------------------------------------- storage
const KEY = 'mbgx-v1-';
/**
* Storage with any userscript manager. Violentmonkey, Tampermonkey and AdGuard offer
* GM_getValue/GM_setValue, answered at once. Greasemonkey 4, Userscripts (Safari) and FireMonkey
* offer only GM.getValue/GM.setValue, which answer later (promises): there every stored value is
* read once at start (loadStorage) into a copy here, kept as JSON text so each read is a fresh copy
* as with GM_getValue; each write goes to the copy and to the manager.
*/
const STORE_KEYS = ['settings', 'seeds', 'allow', 'categories', 'archives', 'logs', 'sessions', 'refSort'];
const gmAsync = typeof GM === 'object' && GM && typeof GM.getValue === 'function' && typeof GM.setValue === 'function' ? GM : null;
const gmSync = typeof GM_getValue === 'function' && typeof GM_setValue === 'function';
const copy = new Map(); // key -> JSON text (promise-based managers only)
async function loadStorage() {
if (gmSync || !gmAsync) return;
let names = STORE_KEYS.map(k => KEY + k);
try { if (typeof gmAsync.listValues === 'function') names = (await gmAsync.listValues()).filter(k => typeof k === 'string' && k.startsWith(KEY)); } catch (e) { /* keep the known keys */ }
await Promise.all(names.map(async k => {
try { const v = await gmAsync.getValue(k); if (v !== undefined) copy.set(k, JSON.stringify(v)); } catch (e) { /* left at its default */ }
}));
}
const read = (k, d) => {
try {
if (gmSync) { const v = GM_getValue(KEY + k, d); return v === undefined ? d : v; }
const t = copy.get(KEY + k);
return t === undefined ? d : JSON.parse(t);
} catch (e) { return d; }
};
const write = (k, v) => {
if (gmSync) return GM_setValue(KEY + k, v);
copy.set(KEY + k, JSON.stringify(v));
if (gmAsync) gmAsync.setValue(KEY + k, JSON.parse(JSON.stringify(v))).catch(e => console.warn('[Better Filter] could not save', k, e));
};
// archivePick: per-board archive choices, { board: { post: host, search: host } }; empty = automatic.
// exportFields: last choices in the export windows (see blacklistFile(), falsePositivesFile()); all off = hashes only.
const DEFAULTS = { enabled: true, stubs: true, archiveUpdates: true, archivePick: {},
exportFields: { references: { source: false, post: false, boards: true, dates: false, cats: false, pieces: true }, falsePositives: { post: false, match: false, dates: false } },
lastCat: '', hideReplies: false,
// editedCopies: every extended check (cropped, bordered, mirrored, collaged, inverted, retouched);
// closeVariants = opt-in: also pictures that are only very similar (another shot of the same scene).
editedCopies: true, closeVariants: false,
// md5Match: exact-file matching. md5Dest: whether "⊘ blacklist" also puts a file's MD5 in a 4chan
// X-family script's filter ('native' | 'both'). xAuto: also add MD5s of files matched by picture to
// their filter (opt-in); xNoStub: rules added there get ";stub:no".
md5Match: true, md5Dest: 'native', xAuto: false, xNoStub: false,
// keepPost: blacklisting keeps the post's number (and thread), for a link to its archived copy (the last choice is remembered)
keepPost: true,
// pieces: heavily cropped copies, by the pieces of each reference (only with editedCopies on)
pieces: true,
// theme: window colours, 'page' (match the page) | 'dark' | 'light'
theme: 'page',
// completeMd5: an exact-file reference that hides a post gets that post's picture (see completeFromPost)
completeMd5: true,
// keepLog: keep the diagnostic log and page summaries in storage across pages (off: this page's are kept in memory only)
keepLog: false,
// workers: check edited copies in background workers where the browser allows them (startWorkers)
workers: true,
a: 4, d: 5, mode: 'both', margin: 4, verbose: false };
const LOG_MAX = 1500, SESSION_MAX = 30;
await loadStorage(); // promise-based managers: everything stored, before anything is read
const storedCfg = read('settings', {}) || {};
let cfg = { ...DEFAULTS, ...storedCfg };
// Settings are kept as they were saved, checked here.
if (!['native', 'both'].includes(cfg.md5Dest)) cfg.md5Dest = 'native';
for (const k of ['enabled', 'stubs', 'archiveUpdates', 'hideReplies', 'editedCopies', 'closeVariants', 'md5Match', 'xAuto', 'xNoStub', 'pieces', 'verbose', 'keepPost', 'keepLog', 'completeMd5', 'workers']) {
if (typeof cfg[k] !== 'boolean') cfg[k] = DEFAULTS[k];
}
if (!['page', 'dark', 'light'].includes(cfg.theme)) cfg.theme = 'page';
if (!cfg.archivePick || typeof cfg.archivePick !== 'object' || Array.isArray(cfg.archivePick)) cfg.archivePick = {};
if (!cfg.exportFields || typeof cfg.exportFields !== 'object' || Array.isArray(cfg.exportFields)) cfg.exportFields = {};
/**
* A category's own general and detection settings, each on or off; one left out follows the
* general setting. x*: the 4chan X-family options (only for this browser: never exported or imported).
*/
const CAT_SETTINGS = [
['stubs', 'general', 'Show a stub for filtered posts', () => !!cfg.stubs],
['hideReplies', 'general', 'Also hide replies to filtered posts', () => !!cfg.hideReplies],
['xAdd', 'x', 'Blacklisting a file into it also adds it to their MD5 filter', () => cfg.md5Dest === 'both'],
['xAuto', 'x', 'Also add files it catches on its own to their MD5 filter', () => !!cfg.xAuto],
['xNoStub', 'x', 'Lines added there hide posts without a stub', () => !!cfg.xNoStub],
['md5', 'detection', 'Exact files (by MD5)', () => !!cfg.md5Match],
['edited', 'detection', 'Edited copies', () => !!cfg.editedCopies],
['pieces', 'detection', 'Including heavily cropped copies', () => cfg.pieces !== false],
['close', 'detection', 'Close variants', () => !!cfg.closeVariants],
];
// (CAT_SETTINGS too: cleanCat() uses it when the categories are first loaded.)
// At most TAG_LIMIT tags per reference, each up to TAG_CHARS characters (cleanSeed keeps no more).
const TAG_LIMIT = 20, TAG_CHARS = 40, TAGS_SHOWN = 6;
// Before the first loadCats(): cleanRules() needs it.
const RULE_LIMIT = 15;
let seeds = loadSeeds(), allow = loadAllow(), cats = loadCats();
// In-memory only: the prepared reference database and whitelist set, rebuilt by prepareData().
let refDB = prepareReferences([]), allowSet = new Set();
let archives = loadArchives();
function loadSeeds() {
const v = read('seeds', []);
return Array.isArray(v) ? v.filter(s => s && typeof s.id === 'string') : [];
}
const isMd5Only = s => !isPicture(s) && !!normMd5(s && s.md5);
/**
* Categories: [{id: 'c1', name, enabled, boards: [], rules: null | {a, d, mode, margin}}]. A
* reference's `cat` names one; references without one are "Uncategorised" (general rules, all
* boards unless they have their own).
*/
function loadCats() {
const v = read('categories', []);
return (Array.isArray(v) ? v : []).filter(c => c && /^c\d+$/.test(c.id) && typeof c.name === 'string').map(cleanCat);
}
function cleanRules(r) {
if (!r || typeof r !== 'object') return null;
const n = v => Math.max(0, Math.min(RULE_LIMIT, Math.round(Number(v)) || 0));
return { a: n(r.a), d: n(r.d), mode: r.mode === 'either' ? 'either' : 'both', margin: n(r.margin) };
}
function cleanCatSettings(c) {
const out = {}, src = c.settings && typeof c.settings === 'object' ? c.settings : {};
for (const [k] of CAT_SETTINGS) if (typeof src[k] === 'boolean') out[k] = src[k];
return Object.keys(out).length ? out : null;
}
/** A setting for posts of this category (id or category): its own if it has one, else the general one. */
function catSetting(cat, key) {
const c = typeof cat === 'string' ? catById(cat) : cat;
if (c && c.settings && typeof c.settings[key] === 'boolean') return c.settings[key];
const def = CAT_SETTINGS.find(([k]) => k === key);
return def ? def[3]() : false;
}
function cleanCat(c) {
const out = { id: String(c.id), name: String(c.name).trim().slice(0, 60) || 'Category', enabled: c.enabled !== false };
const b = Array.isArray(c.boards) ? [...new Set(c.boards.filter(v => /^[a-z0-9]{1,12}$/.test(v)))] : [];
if (b.length) out.boards = b;
const r = cleanRules(c.rules);
if (r) out.rules = r;
const set = cleanCatSettings(c);
if (set) out.settings = set;
return out;
}
/** Whitelist entries: hashes, an MD5, or both. */
function loadAllow() {
const v = read('allow', []);
return (Array.isArray(v) ? v : []).filter(x => x && typeof x === 'object' && (typeof x.a === 'string' || normMd5(x.md5)));
}
/** UTC calendar date only: blacklists are shareable, so no time of day or timezone. */
const today = () => new Date().toISOString().slice(0, 10);
/**
* A reference reduced to what matching and archive links need, with a date-only "added".
* Used for everything stored, imported or exported, so a shared list carries nothing else.
*/
function cleanSeed(x) {
// A picture reference has the hash method and both hashes; an exact-file one only its MD5.
const s = { id: String(x.id).slice(0, 40) };
if (isPicture(x)) Object.assign(s, { algo: x.algo, a: x.a, d: x.d });
if (Number.isInteger(x.w) && Number.isInteger(x.h)) { s.w = x.w; s.h = x.h; }
if (typeof x.added === 'string' && /^\d{4}-\d{2}-\d{2}/.test(x.added)) s.added = x.added.slice(0, 10);
if (/^[a-z0-9]{1,12}$/.test(x.board)) s.board = x.board;
for (const k of ['thread', 'post']) if (/^\d{1,12}$/.test(String(x[k]))) s[k] = String(x[k]);
if (typeof x.thumb === 'string' && /^https:\/\/i\.4cdn\.org\/\w+\/\d+s\.jpg$/.test(x.thumb)) s.thumb = x.thumb;
if (normMd5(x.md5)) s.md5 = normMd5(x.md5);
if (s.a && DETAIL_TEXT.test(x.detail || '')) s.detail = x.detail;
if (s.a && x.blank === true) s.blank = true; // made from a nearly blank thumbnail, on purpose
const pt = x.part;
if (s.a && pt && HEX64.test(pt.a) && HEX64.test(pt.d) && Number.isInteger(pt.w) && Number.isInteger(pt.h) && DETAIL_TEXT.test(pt.detail || '')) {
s.part = { a: pt.a, d: pt.d, w: pt.w, h: pt.h, detail: pt.detail };
}
// pieces: stored as text (packPieces())
const pcText = s.a && x.pieces && piecesText(x.pieces);
if (pcText) s.pieces = pcText;
if (Array.isArray(x.boards)) {
const b = [...new Set(x.boards.filter(v => /^[a-z0-9]{1,12}$/.test(v)))];
if (b.length) s.boards = b;
}
if (typeof x.cat === 'string' && /^c\d+$/.test(x.cat)) s.cat = x.cat;
if (Array.isArray(x.tags)) {
const t = [...new Set(x.tags.filter(v => typeof v === 'string').map(v => v.trim().slice(0, TAG_CHARS)).filter(Boolean))].slice(0, TAG_LIMIT);
if (t.length) s.tags = t;
}
return s;
}
// The references in use (id -> position; -1: by MD5 only): a post hidden by one that is gone, or no longer
// applies on its board, is shown at once, before it is checked again (refApplies()).
let refIds = new Map();
const refApplies = (id, board) => {
const i = refIds.get(id);
if (i === undefined || i < 0) return i !== undefined;
const idx = boardIndex(refDB, board);
let lo = 0, hi = idx.length - 1;
while (lo <= hi) { const m = (lo + hi) >> 1; if (idx[m] === i) return true; if (idx[m] < i) lo = m + 1; else hi = m - 1; }
return false;
};
/** Rebuild the in-memory matching data from `seeds` and `allow` (after any change to either). */
function prepareData() {
const t0 = performance.now();
refDB = prepareReferences(seeds, HASH_ALGO, cats, { md5: !!cfg.md5Match, edited: !!cfg.editedCopies, pieces: cfg.pieces !== false, close: !!cfg.closeVariants });
refIds = new Map();
refDB.seeds.forEach((s, i) => { if (!refIds.has(s.id)) refIds.set(s.id, i); });
for (const s of refDB.md5Seeds) if (!refIds.has(s.id)) refIds.set(s.id, -1);
allowSet = prepareAllow(allow);
stats.prepMs = round(performance.now() - t0);
if ((pool.state === 'on' || pool.state === 'starting') && workerLimit()) stopWorkers('off', workerLimit()); // the list grew too long
else if (pool.state === 'off' && pool.reason && pool.reason !== 'turned off' && !workerLimit()) pool.reason = ''; // short enough again
if (pool.state === 'on') sendRefs();
else buildOverlapIndex(refDB);
if (refDB.anyExt || refDB.anyClose) startWorkers();
}
/**
* The overlapping-crop index (overlapIndexStep()) is built between other work, ~30 ms at a time, so a
* long list never holds the page up. Pictures checked before it was ready are checked again with it.
*/
function buildOverlapIndex(db) {
if (!db.anyExt || db.ov || db.ovBuilding) return; // built, or being built
db.ovBackground = true;
db.ovBuilding = true;
const t0 = performance.now();
const step = () => guard('overlap-index', () => {
// the references changed meanwhile (that database has its own build), or the worker matches: stopped (it can start again)
if (db !== refDB || pool.live) { db.ovBuilding = false; return; }
if (!overlapIndexStep(db, 30)) { setTimeout(step, 0); return; }
db.ovBuilding = false;
stats.overlapIndexMs = round(performance.now() - t0);
for (const st of posts.values()) {
if (st.ovPending && st.root && st.root.isConnected && !(st.result && st.result.match)) { st.ovPending = false; guard('applyMatch', () => applyMatch(st)); }
}
});
setTimeout(step, 0);
}
// ---------------------------------------------------------------- background workers
/**
* Edited-copy matching (retouched copies, every edited-copy window, overlapping crops, close variants)
* runs in background workers where the browser allows them: the page stays free however heavy the
* checks or long the list, and several pictures are checked at once. Each worker runs the same matching
* code (mbgxCore()) on its own copy of the reference database, built from the same list in the same
* order, so a result names its reference by its position. Each list gets a version: a result for an
* older one is dropped and the picture checked again. Until the first worker has its database ready, the
* page checks the pictures in view itself, so they are not kept waiting. Without workers (not supported, blocked by the
* page, not answering, or turned off) the same checks run on the page as before.
*/
// live: a worker has its database ready (until then the page checks the pictures in view itself, so they are not kept waiting)
const pool = { workers: [], state: 'off', live: false, reason: '', version: 0, jobs: new Map(), waiting: [], nextId: 1, indexMs: 0, retired: 0, watchdog: 0, watched: 0, url: '', expected: null };
// Pictures a worker may hold at once (the rest wait here); how long a worker may go silent while it has work: per
// picture, and while it prepares a new list (very long lists on slow devices take a while).
const WORKER_BACKLOG = 32, WORKER_JOB_MS = 8000, WORKER_PREPARE_MS = 180000, WORKER_START_MS = 10000;
/**
* A small fixed check of the matching code: a drawn test picture as the only reference, then a mirror image
* and a closer crop of it (both must be found) and a different picture (must not be). Each worker runs it
* before it gets any work, and its answer must be exactly the page's: a worker that starts but cannot
* match is never used.
*/
function matchSelfTest(C) {
const W = 96, H = 128, pic = f => { const g = new Uint8ClampedArray(W * H); for (let y = 0; y < H; y++) for (let x = 0; x < W; x++) g[y * W + x] = f(x, y); return { g, w: W, h: H }; };
const base = pic((x, y) => 128 + 90 * Math.sin(x / 9) * Math.cos(y / 13) + (x > 60 && y < 40 ? 60 : 0) - (x < 30 && y > 90 ? 50 : 0));
const fp = C.hashGray(Float64Array.from(base.g), W, H), pcs = C.piecesOf(base, fp);
const db = C.prepareReferences([{ id: 't1', algo: fp.algo, a: fp.a, d: fp.d, w: W, h: H, detail: C.detailOf(C.integral(base)), pieces: C.packPieces(pcs) }]);
const ways = { crop: 1, border: 1, mirror: 1, panels: 1, invert: 1, zoom: 1, tilt: 1, turn: 1 }, rule = { a: 4, d: 5, mode: 'both', margin: 4 };
const mirror = pic((x, y) => base.g[y * W + W - 1 - x]);
const crop = pic((x, y) => base.g[Math.floor(10 + y * 0.8) * W + Math.floor(8 + x * 0.8)]);
const other = pic((x, y) => 128 + 100 * Math.cos(x / 5 + y / 7));
const one = px => { const f = C.matchExtended(px, db, rule, 'test', ways) || C.matchOverlap(px, db, rule, 'test'); return f ? [f.how, Math.round(f.a64 * 10), Math.round(f.d64 * 10), f.corr == null ? null : Math.round(f.corr * 1000)] : null; };
return JSON.stringify([one(mirror), one(crop), one(other)]);
}
/** What each background worker runs, with the matching code (mbgxCore()) as C. */
function workerMain(C) {
let db = null, version = -1, cfg = null;
self.onmessage = e => {
const m = e.data;
try {
if (m.type === 'ping') { self.postMessage({ type: 'pong', test: matchSelfTest(C) }); return; }
if (m.type === 'refs') {
const t0 = performance.now();
version = m.version;
cfg = m.cfg;
db = C.prepareReferences(m.seeds, C.HASH_ALGO, m.cats, m.opts);
if (db.anyExt) C.overlapIndex(db);
self.postMessage({ type: 'ready', version, ms: performance.now() - t0 });
} else if (m.type === 'detail') {
if (m.version === version && db && db.seeds[m.i]) { db.seeds[m.i].detail = m.detail; db.details.delete(m.i); }
} else if (m.type === 'match') {
if (m.version !== version || !db) { self.postMessage({ type: 'result', id: m.id, version, stale: true }); return; }
const t0 = performance.now(), px = { g: m.g, w: m.w, h: m.h }, ext = m.ext;
const look = ext ? C.matchLookalike(px, m.fp, db, cfg, m.board, C.LOOK) : null;
const lookMs = performance.now() - t0, count = { windows: 0, aligns: 0, evals: 0 };
const x = !look && ext ? C.matchExtended(px, db, cfg, m.board, ext, count) || (ext.overlap ? C.matchOverlap(px, db, cfg, m.board, count) : null) : null;
const close = !look && !x && db.anyClose ? C.matchLookalike(px, m.fp, db, cfg, m.board, C.CLOSE) : null;
const f = look || x || close;
self.postMessage({ type: 'result', id: m.id, version, ms: performance.now() - t0, lookMs, windows: count.windows, aligns: count.aligns,
hit: f && { kind: look ? 'retouch' : x ? 'edited' : 'close', how: x ? x.how : null, i: db.seeds.indexOf(f.seed),
da: f.da, dd: f.dd, a64: f.a64, d64: f.d64, corr: f.corr } });
}
} catch (err) {
self.postMessage({ type: 'error', id: m.id, error: String(err && err.stack || err) });
}
};
}
/**
* One worker: it keeps up with fetching and hashing, and each worker holds its own copy of the list and its
* crop index, so more would mostly cost memory.
*/
function workerCount() {
return 1;
}
/**
* Why the worker is not used for this list, or ''. Its copy of the list and crop index grows with the list, so a
* very long list (more on a phone) is checked on the page instead, as without workers.
*/
function workerLimit() {
const most = /Android|iPhone|iPad|Mobile/i.test(navigator.userAgent) ? 5000 : 20000;
return seeds.length > most ? `not used for lists over ${most.toLocaleString('en')} references` : '';
}
function startWorkers() {
if (pool.state !== 'off') return;
if (cfg.workers === false) { pool.reason = 'turned off'; return; }
if (workerLimit()) { pool.reason = workerLimit(); return; }
if (typeof Worker !== 'function' || typeof Blob !== 'function') { pool.state = 'failed'; pool.reason = 'not supported here'; return; }
pool.state = 'starting';
pool.reason = '';
try {
const code = `'use strict';\nconst C = (${mbgxCore.toString()})();\nconst matchSelfTest = ${matchSelfTest.toString()};\n(${workerMain.toString()})(C);\n`;
const url = pool.url = URL.createObjectURL(new Blob([code], { type: 'text/javascript' }));
for (let k = 0, n = workerCount(); k < n; k++) {
const entry = { w: new Worker(url), busy: 0, ok: false, ready: 0, heard: performance.now(), preparing: false };
entry.w.onmessage = e => guard('worker', () => onWorkerMessage(entry, e.data));
entry.w.onerror = e => guard('worker', () => { if (e && e.preventDefault) e.preventDefault(); workerLost(entry, e && e.message ? e.message : 'blocked or failed to start'); });
entry.w.onmessageerror = () => guard('worker', () => workerLost(entry, 'a message could not be read'));
pool.workers.push(entry);
entry.w.postMessage({ type: 'ping' });
}
} catch (e) {
workersFailed(String(e && e.message || e));
return;
}
// the page's own answer to the self-test, once, as a task of its own (not added to this one)
setTimeout(() => guard('worker', selfTestAnswer), 0);
// a page can refuse workers without an exception: they then never answer
const started = pool.workers.slice();
afterRunning(WORKER_START_MS, () => guard('worker', () => { if (pool.state === 'starting' && pool.workers.length && pool.workers.every(x => started.includes(x))) workersFailed('no answer'); }));
if (!pool.watchdog) { pool.watched = performance.now(); pool.watchdog = setInterval(() => guard('worker', watchWorkers), 2000); }
}
/**
* A worker that has work but has gone silent for too long (WORKER_JOB_MS per picture, WORKER_PREPARE_MS while
* it prepares a new list): every worker is stopped and the page checks the pictures itself, as something in
* this browser is not right. Time the page itself was paused does not count (afterRunning()).
*/
function watchWorkers() {
const now = performance.now(), paused = now - pool.watched - 2500;
pool.watched = now;
for (const entry of pool.workers) {
if (paused > 0) entry.heard += paused;
const limit = entry.preparing ? WORKER_PREPARE_MS : entry.busy > 0 ? WORKER_JOB_MS : 0;
if (limit && now - entry.heard > limit) { workersFailed('stopped answering'); break; }
}
if (pool.state !== 'on' && pool.state !== 'starting') { clearInterval(pool.watchdog); pool.watchdog = 0; }
}
/** The worker's code, as an address, is released once every worker has loaded it (or none will). */
function dropWorkerCode() {
if (pool.url) { try { URL.revokeObjectURL(pool.url); } catch (e) { /* already released */ } pool.url = ''; }
}
/** The page's answer to the self-test (matchSelfTest()), computed once. */
function selfTestAnswer() {
if (pool.expected === null) pool.expected = matchSelfTest({ hashGray, piecesOf, prepareReferences, detailOf, integral, packPieces, matchExtended, matchOverlap });
return pool.expected;
}
/** The reference list (and the settings it is judged by) to every worker, as a new version. */
function sendRefs() {
pool.version++;
const msg = { type: 'refs', version: pool.version, seeds, cats, opts: { md5: !!cfg.md5Match, edited: !!cfg.editedCopies, pieces: cfg.pieces !== false, close: !!cfg.closeVariants },
cfg: JSON.parse(JSON.stringify(cfg)) };
for (const entry of pool.workers) { entry.w.postMessage(msg); entry.preparing = true; entry.heard = performance.now(); }
}
function onWorkerMessage(entry, m) {
if (!pool.workers.includes(entry)) return; // stopped meanwhile
entry.heard = performance.now();
if (m.type === 'pong') {
if (m.test !== selfTestAnswer()) { workerLost(entry, 'its self-test gave a different answer'); return; }
entry.ok = true;
if (pool.state === 'starting' && pool.workers.length && pool.workers.every(x => x.ok)) {
pool.state = 'on';
dropWorkerCode(); // every worker has loaded it
pool.reason = '';
log('workers-on', { workers: pool.workers.length });
sendRefs();
}
} else if (m.type === 'ready') {
entry.ready = m.version;
if (m.version !== pool.version) return;
entry.preparing = false;
pool.indexMs = Math.max(pool.indexMs, round(m.ms));
if (!pool.live) {
pool.live = true;
// pictures checked on the page before its own crop index was ready go to the workers now
for (const st of posts.values()) {
if (st.ovPending && st.root && st.root.isConnected && !(st.result && st.result.match)) { st.ovPending = false; guard('applyMatch', () => applyMatch(st)); }
}
}
} else if (m.type === 'result' || m.type === 'error') {
const job = pool.jobs.get(m.id);
if (!job) return;
pool.jobs.delete(m.id);
entry.busy--;
if (m.type === 'error') {
log('worker-error', { post: job.st.key, error: String(m.error).slice(0, 300) });
stats.scriptErrors++;
if (job.st.result === job.r) matchOnPage(job.st, job.r);
} else workerResult(job, m);
feedWorkers();
}
}
/** Pictures waiting for room in a worker, while there is room. */
function feedWorkers() {
while (pool.waiting.length && pool.state === 'on') {
const job = pool.waiting.shift();
if (job.st.result !== job.r) continue; // checked again meanwhile
if (!dispatch(job.st, job.r, job.ext)) { pool.waiting.unshift(job); return; }
}
}
/**
* One worker stops (an error, a wrong self-test, or silence): its pictures go to the other workers, or to the
* page when none is left.
*/
function workerLost(entry, reason) {
if (!pool.workers.includes(entry)) return;
pool.workers = pool.workers.filter(x => x !== entry);
pool.retired++;
try { entry.w.terminate(); } catch (e) { /* already gone */ }
log('worker-stopped', { reason, left: pool.workers.length });
if (!pool.workers.length || pool.state === 'starting') { workersFailed(reason); return; }
if (!pool.workers.some(x => x.ready === pool.version)) pool.live = pool.workers.some(x => x.ready > 0);
const jobs = [...pool.jobs].filter(([, job]) => job.entry === entry);
for (const [id, job] of jobs) {
pool.jobs.delete(id);
if (job.st.result === job.r) guard('applyMatch', () => applyMatch(job.st));
}
}
/** Workers refused or failed: they are stopped, and every picture they had is checked on the page instead. */
function workersFailed(reason) {
if (pool.state === 'failed') return;
stopWorkers('failed', reason);
}
/**
* Every worker stopped ('failed', or 'off' when turned off): the pictures they had or that were waiting are
* checked on the page, which builds its own crop index again.
*/
function stopWorkers(state, reason) {
pool.state = state;
pool.live = false;
pool.reason = reason;
for (const entry of pool.workers) { try { entry.w.terminate(); } catch (e) { /* already gone */ } }
pool.workers = [];
dropWorkerCode();
log('workers-off', { reason });
const jobs = [...pool.jobs.values()].concat(pool.waiting);
pool.jobs.clear();
pool.waiting = [];
if (refDB.anyExt && !refDB.ov) buildOverlapIndex(refDB);
for (const job of jobs) if (job.st.result === job.r) guard('applyMatch', () => applyMatch(job.st));
}
/** The Diagnostics switch: workers stop at once (their pictures are checked on the page), or start again. */
function setWorkers(on) {
if (!on && (pool.state === 'on' || pool.state === 'starting')) stopWorkers('off', 'turned off');
else if (!on) pool.reason = 'turned off';
else if (pool.state === 'off' || pool.state === 'failed') { pool.state = 'off'; pool.reason = ''; }
}
/** A picture to a worker, or to the queue when every worker is full (fed as they answer, feedWorkers()). */
function sendToWorkers(st, r, ext) {
if (!dispatch(st, r, ext)) pool.waiting.push({ st, r, ext });
}
/**
* A picture to the least busy worker with the current list (else the least busy); its grey copy is sent as a
* copy (the page keeps its own). Returns false, sending nothing, when every worker is full (WORKER_BACKLOG).
*/
function dispatch(st, r, ext) {
const current = pool.workers.filter(x => x.ready === pool.version);
const entry = (current.length ? current : pool.workers).reduce((x, y) => (y.busy < x.busy ? y : x), { busy: Infinity });
if (!entry.w || entry.busy >= WORKER_BACKLOG) return false;
const id = pool.nextId++, g = st.px.g.slice();
pool.jobs.set(id, { st, r, version: pool.version, entry });
if (!entry.busy) entry.heard = performance.now(); // silence counts from when an idle worker gets work
entry.busy++;
entry.w.postMessage({ type: 'match', id, version: pool.version, g, w: st.px.w, h: st.px.h, fp: { a: st.fp.a, d: st.fp.d, w: st.fp.w, h: st.fp.h }, board: st.board, ext }, [g.buffer]);
return true;
}
function workerResult(job, m) {
const { st, r } = job;
if (st.result !== r) return; // checked again meanwhile: that check decides
if (m.stale || m.version !== pool.version) { guard('applyMatch', () => applyMatch(st)); return; } // an older list
addTiming(stats.lookMs, m.lookMs);
addTiming(stats.extMs, m.ms);
stats.extWindows += m.windows;
stats.extAligns += m.aligns;
const h = m.hit, seed = h && Number.isInteger(h.i) && h.i >= 0 ? refDB.seeds[h.i] : null;
if (h && !seed) { matchOnPage(st, r); return; } // a match that names no reference here: the page decides
if (seed) {
const rule = refDB.ruleIdx[h.i] ? refDB.rules[refDB.ruleIdx[h.i]] : cfg, corr3 = c => (c == null ? null : Math.round(c * 1000) / 1000);
r.match = h.kind === 'edited' ? { seed, da: round(h.a64), dd: round(h.d64), rule, how: h.how, corr: corr3(h.corr) }
: { seed, da: h.da, dd: h.dd, rule, how: h.kind, corr: corr3(h.corr) };
r.near = false;
}
finishMatch(st, r);
}
/** Stored archive table if a valid update was downloaded, otherwise the one built into the script. */
function loadArchives() {
const t = read('archives', null);
// v2 = per-board candidate lists; any other stored table is ignored and refreshed on the next panel open.
return t && t.v === 2 && Array.isArray(t.hosts) && Array.isArray(t.ff) && t.boards && typeof t.boards === 'object' ? t : { ...BUNDLED_ARCHIVES, updated: null };
}
/** References matched by picture (those in enabled categories). */
const activeSeeds = () => refDB.seeds;
// ---------------------------------------------------------------- diagnostics
const session = { id: Date.now().toString(36), url: location.href, start: new Date().toISOString() };
const scriptStart = performance.now(); // ms since the page started loading
const timing = () => ({ n: 0, total: 0, max: 0 });
const stats = {
scans: 0, posts: 0, imagePosts: 0, noImage: 0, clonesSkipped: 0, buttons: 0, alreadyHidden: 0,
queued: 0, hashOK: 0, hashErrors: 0, lowDetail: 0, matched: 0, stubbed: 0, nearMisses: 0,
revealed: 0, whitelisted: 0, scriptErrors: 0, directFetches: 0, fetchMs: timing(), decodeMs: timing(), hashMs: timing(), matchMs: timing(), extMs: timing(), byMethod: {}, prepMs: 0, cacheHits: 0,
xAutoAdded: 0,
// Where the time goes. decodeMs = bitmapMs (waiting for the browser's decoder, and then for
// the page to be free) + drawMs (drawing and reading the pixels); extMs includes lookMs (retouched).
// queueMs: queued until fetched; decisionMs: post found until decided (hashed posts), visibleDecisionMs
// the same for posts in view when queued. stalls: main-thread delays of more than 50 ms during a batch.
thumbBytes: 0, bitmapMs: timing(), drawMs: timing(), lookMs: timing(), extWindows: 0, extAligns: 0,
queueMs: timing(), decisionMs: timing(), visibleDecisionMs: timing(), stalls: timing(),
peakDecodes: 0, canvases: 0, bitmaps: 0, largestPixels: 0,
};
// The last 512 values of each timing, for percentiles in the export and the panel (not stored per value).
const SAMPLE_N = 512, samples = new Map();
let pendingLogs = [], flushTimer = 0;
const pageLog = []; // this page's entries, in memory only (the full export's log when keepLog is off)
function log(type, data = {}) {
const entry = { t: new Date().toISOString(), s: session.id, type, ...data };
pageLog.push(entry);
if (pageLog.length > LOG_MAX) pageLog.splice(0, pageLog.length - LOG_MAX);
if (!cfg.keepLog) return; // nothing goes to storage unless asked for
pendingLogs.push(entry);
if (pendingLogs.length > LOG_MAX) pendingLogs.splice(0, pendingLogs.length - LOG_MAX);
if (!flushTimer) flushTimer = setTimeout(flush, 3000);
}
/** Logs are buffered and merged into storage at most every 3 s (and on page hide), not per event; only with keepLog on. */
function flush() {
clearTimeout(flushTimer);
flushTimer = 0;
if (!cfg.keepLog) { pendingLogs = []; return; }
try {
if (pendingLogs.length) {
write('logs', read('logs', []).concat(pendingLogs).slice(-LOG_MAX));
pendingLogs = [];
}
const sessions = read('sessions', []).filter(s => s.id !== session.id);
sessions.push({ ...session, end: new Date().toISOString(), stats: statsOut() });
write('sessions', sessions.slice(-SESSION_MAX));
} catch (e) {
console.warn('[Better Filter] could not save diagnostics', e);
}
}
function addTiming(t, ms) {
t.n++; t.total += ms; t.max = Math.max(t.max, ms);
let s = samples.get(t);
if (!s) samples.set(t, s = new Float32Array(SAMPLE_N));
s[(t.n - 1) % SAMPLE_N] = ms;
}
const round = ms => Math.round(ms * 10) / 10;
/** Median and 95th percentile of a timing's recent values (null before any). */
function percentiles(t) {
const s = samples.get(t);
if (!s || !t.n) return { p50: null, p95: null };
const v = s.slice(0, Math.min(t.n, SAMPLE_N)).sort();
return { p50: round(v[v.length >> 1]), p95: round(v[Math.min(v.length - 1, Math.floor(v.length * 0.95))]) };
}
/** The page's stats as stored and exported: each timing with its percentiles, totals rounded. */
function statsOut() {
const out = {};
for (const [k, v] of Object.entries(stats)) {
out[k] = v && typeof v === 'object' && 'total' in v && 'n' in v ? { n: v.n, total: round(v.total), max: round(v.max), ...percentiles(v) } : v;
}
return out;
}
// This page's script errors for the panel, grouped by where + message (at most 20 distinct).
const recentErrors = [];
const ERROR_PLACES = { command: 'clicking a Better Filter control', scan: 'scanning the page for posts', attach: 'setting up a post',
applyMatch: 'checking a post against references', rematch: 're-checking posts', panel: 'opening the panel',
control: 'adding the settings icon', 'overlap-index': 'preparing the checks for overlapping crops', start: 'starting up', quotes: 'updating links and replies to filtered posts', details: 'saving reference details' };
function guard(where, fn) {
try {
return fn();
} catch (e) {
stats.scriptErrors++;
const msg = String(e && e.message || e);
const known = recentErrors.find(x => x.where === where && x.error === msg);
if (known) known.count++;
else if (recentErrors.length < 20) recentErrors.push({ where, error: msg, count: 1 });
log('script-error', { where, error: String(e), stack: String(e && e.stack || '').split('\n').slice(0, 5).join(' | ') });
console.error('[Better Filter]', where, e);
}
}
function environment() {
const cl = document.documentElement.classList;
let manager = 'unknown', injectInto = null;
try {
const info = typeof GM_info === 'object' && GM_info ? GM_info : gmAsync && gmAsync.info; // GM.info: Greasemonkey 4
manager = `${info.scriptHandler} ${info.version}`; injectInto = info.injectInto || null;
} catch (e) { /* no info from the manager */ }
manager += gmSync ? ' (GM_ storage)' : gmAsync ? ' (GM. storage)' : ' (no storage: settings last until the page closes)';
// injectInto (Violentmonkey): "page" runs the script in the page's own realm, "content" in a sandbox
// where reading the canvas's pixels goes through wrappers (slower hashing).
// style scripts: OneeChan marks <html>; its fork StyleChan doesn't, but leaves its theme colours (--sc-*)
const styleChan = !!document.getElementById('sc-theme-vars') || getComputedStyle(document.documentElement).getPropertyValue('--sc-mainColor') !== '';
return { fourchanXT: cl.contains('fourchan-xt'), fourchanX: cl.contains('fourchan-x'), xFamily: xFamily(), oneechan: cl.contains('oneechan') && !styleChan,
styleScript: styleChan ? 'StyleChan' : cl.contains('oneechan') ? 'OneeChan' : null,
manager, injectInto, timerStepMs: timerStep(), userAgent: navigator.userAgent };
}
/**
* Smallest step of performance.now() here (browsers may round it, e.g. to 1 ms): how fine single
* timings are. Measured once (it waits for up to five clock ticks).
*/
let timerStepMs;
function timerStep() {
if (timerStepMs !== undefined) return timerStepMs;
let step = Infinity;
for (let k = 0, t = performance.now(); k < 5; k++) {
let u = performance.now();
for (let i = 0; i < 1e6 && u === t; i++) u = performance.now();
if (u > t) step = Math.min(step, u - t);
t = u;
}
return (timerStepMs = step === Infinity ? null : Math.round(step * 1000) / 1000);
}
/**
* What the browser itself reports, where it can, all measured, none estimated:
* transfer - direct thumbnail fetches as Resource Timing sees them: their body bytes, and how many
* were likely served from the browser cache (no bytes transferred and done within
* 15 ms; Firefox reports no transfer size for 4chan's thumbnails even over the network).
* "hidden": sizes not shown at all (a cross-origin server's choice).
* longTasks - main-thread tasks over 50 ms (Chromium; Firefox has no such API: null).
* jsHeap - highest JavaScript heap size seen (Chromium's performance.memory; Firefox: null).
*/
const browserStats = { transfer: { fetches: 0, bodyBytes: 0, likelyFromCache: 0, hidden: 0 }, longTasks: null, jsHeapPeak: null };
function watchBrowserStats() {
const types = (typeof PerformanceObserver === 'function' && PerformanceObserver.supportedEntryTypes) || [];
try {
if (types.includes('resource')) {
new PerformanceObserver(list => {
for (const e of list.getEntries()) {
if (e.initiatorType !== 'fetch' || !/^https:\/\/i\.4cdn\.org\/\w+\/\d+s\.jpg$/.test(e.name)) continue;
const t = browserStats.transfer;
t.fetches++;
if (!e.encodedBodySize) { t.hidden++; continue; }
t.bodyBytes += e.encodedBodySize;
if (!e.transferSize && e.duration < 15) t.likelyFromCache++;
}
}).observe({ type: 'resource', buffered: true });
}
if (types.includes('longtask')) {
browserStats.longTasks = timing();
new PerformanceObserver(list => { for (const e of list.getEntries()) addTiming(browserStats.longTasks, e.duration); }).observe({ type: 'longtask' });
}
} catch (e) { /* observers are optional */ }
}
/** Chromium only (performance.memory): sampled when a thumbnail is decided, so no timer of its own. */
function sampleHeap() {
const m = performance.memory;
if (m && m.usedJSHeapSize) browserStats.jsHeapPeak = Math.max(browserStats.jsHeapPeak || 0, m.usedJSHeapSize);
}
/** Detects canvas readback being blocked or noised (e.g. Firefox privacy.resistFingerprinting). */
function canvasSelfTest() {
try {
const c = document.createElement('canvas');
c.width = 4; c.height = 2;
const x = c.getContext('2d', { willReadFrequently: true });
x.fillStyle = 'rgb(10,200,30)'; x.fillRect(0, 0, 2, 2);
x.fillStyle = 'rgb(240,20,120)'; x.fillRect(2, 0, 2, 2);
const px = x.getImageData(0, 0, 4, 2).data;
let maxDev = 0;
for (let i = 0; i < 8; i++) {
const want = (i % 4) < 2 ? [10, 200, 30] : [240, 20, 120];
for (let ch = 0; ch < 3; ch++) maxDev = Math.max(maxDev, Math.abs(px[i * 4 + ch] - want[ch]));
}
return { ok: maxDev <= 2, maxDev };
} catch (e) {
return { ok: false, error: String(e) };
}
}
// ---------------------------------------------------------------- thumbnail retrieval
/**
* fn() once the page has been running for `ms`: time it was paused (a background tab the browser froze, a
* computer asleep, a long task) counts at most 1.5 s per second-long step, so whatever arrived meanwhile is
* handled before anything is given up on. Returns a function that cancels it.
*/
function afterRunning(ms, fn) {
let t, used = 0, last = performance.now();
const tick = () => {
const now = performance.now();
used += Math.min(now - last, 1500);
last = now;
if (used >= ms) fn(); else t = setTimeout(tick, Math.min(1000, ms - used));
};
t = setTimeout(tick, Math.min(1000, ms));
return () => clearTimeout(t);
}
function withTimeout(promise, ms, what) {
let cancel;
return Promise.race([promise, new Promise((_, reject) => {
cancel = afterRunning(ms, () => reject(new Error(`${what} timed out after ${ms} ms`)));
})]).finally(() => cancel());
}
let directFetch = true;
/**
* Thumbnail bytes. i.4cdn.org's CORS headers allow a direct fetch from the page, which is much
* quicker than a userscript manager's request. If a direct fetch
* fails for any reason other than an HTTP error, the GM request is used for the rest of the page.
*/
async function fetchBlob(url) {
if (!allowedUrl(url)) throw new Error('not a 4chan thumbnail address'); // nothing else is ever requested
if (directFetch && /^https:\/\/i\.4cdn\.org\//.test(url)) {
let r;
try {
r = await fetch(url, { credentials: 'omit' });
} catch (e) {
directFetch = false;
log('direct-fetch-disabled', { url, error: String(e && e.message || e) });
}
if (r) {
if (!r.ok) throw new Error(`HTTP ${r.status}`);
stats.directFetches++;
return r.blob();
}
}
return gmRequestBlob(url);
}
/**
* The only addresses Better Filter requests on its own: 4chan thumbnails, and the archive list.
* Checked before every request, whatever a page (or another script changing it) puts in a
* thumbnail's place. An archive lookup you start has its own, as strict, check (lookupInArchive()).
*/
const allowedUrl = url => typeof url === 'string' && (/^https:\/\/i\.4cdn\.org\/[a-z0-9]{1,12}\/\d{1,20}s\.jpg$/.test(url) || url === ARCHIVES_URL);
/**
* Privileged request (no CORS restrictions, no cookies); falls back to fetch() if no GM request API
* exists. `allowed` checks the address first: 4chan thumbnails and the archive list, or for an archive
* lookup you started, that archive's search and thumbnails (lookupInArchive()).
*/
function gmRequestBlob(url, allowed = allowedUrl) {
if (!allowed(url)) return Promise.reject(new Error('not an address Better Filter requests'));
const gmx = typeof GM_xmlhttpRequest === 'function' ? GM_xmlhttpRequest
: (typeof GM !== 'undefined' && GM && GM.xmlHttpRequest);
if (!gmx) {
return fetch(url, { credentials: 'omit' }).then(r => {
if (!r.ok) throw new Error(`HTTP ${r.status}`);
return r.blob();
});
}
return new Promise((resolve, reject) => {
gmx({
method: 'GET', url, responseType: 'blob', timeout: 15000, anonymous: true,
onload: r => (r.status >= 200 && r.status < 300 && r.response)
? resolve(new Blob([r.response])) // re-wrap: normalises cross-realm Blobs / ArrayBuffers
: reject(new Error(`HTTP ${r.status}`)),
onerror: r => reject(new Error('network error' + (r && r.error ? `: ${r.error}` : ''))),
ontimeout: () => reject(new Error('request timeout')),
onabort: () => reject(new Error('request aborted')),
});
});
}
// One canvas for every decode: drawing a thumbnail and reading its pixels back is synchronous,
// so decodes in flight at the same time can't interleave on it. Created on first use.
let decodeCanvas = null, decodeCtx = null, activeDecodes = 0;
/** Pixels of a thumbnail: {rgba, w, h, bitmapMs (waiting for the decoded image), drawMs (canvas work)}. */
async function decodePixels(blob) {
stats.peakDecodes = Math.max(stats.peakDecodes, ++activeDecodes);
try {
return await decodeOnce(blob);
} finally {
activeDecodes--;
}
}
async function decodeOnce(blob) {
let src, w, h, cleanup;
const t0 = performance.now();
try {
src = await createImageBitmap(blob);
stats.bitmaps++;
w = src.width; h = src.height;
cleanup = () => src.close();
} catch (e) {
const url = URL.createObjectURL(blob);
cleanup = () => URL.revokeObjectURL(url);
src = await new Promise((resolve, reject) => {
const im = new Image();
im.onload = () => resolve(im);
im.onerror = () => reject(new Error(`image decode failed (${e && e.message || e})`));
im.src = url;
});
w = src.naturalWidth; h = src.naturalHeight;
}
const t1 = performance.now();
try {
if (!w || !h) throw new Error('decoded image has no size');
// Thumbnails are at most 250 px; cap anything larger so a full-size image can't stall the page.
const scale = Math.min(1, 512 / Math.max(w, h));
const cw = Math.max(1, Math.round(w * scale)), ch = Math.max(1, Math.round(h * scale));
if (!decodeCanvas) {
decodeCanvas = document.createElement('canvas');
decodeCtx = decodeCanvas.getContext('2d', { willReadFrequently: true });
stats.canvases++;
}
const c = decodeCanvas, x = decodeCtx;
if (c.width !== cw || c.height !== ch) { c.width = cw; c.height = ch; } // resizing also resets the context
x.fillStyle = '#fff';
x.fillRect(0, 0, cw, ch);
x.drawImage(src, 0, 0, cw, ch);
const rgba = x.getImageData(0, 0, cw, ch).data;
stats.largestPixels = Math.max(stats.largestPixels, cw * ch);
return { rgba, w: cw, h: ch, bitmapMs: t1 - t0, drawMs: performance.now() - t1 };
} finally {
cleanup();
}
}
async function fingerprint(url) {
const t0 = performance.now();
return hashBlob(await withTimeout(fetchBlob(url), 20000, 'thumbnail request'), t0);
}
async function hashBlob(blob, t0 = performance.now()) {
if (blob.size > 2 * 1024 * 1024) throw new Error(`not a thumbnail (${Math.round(blob.size / 1024)} KB)`); // 4chan's are a few KB
const t1 = performance.now();
const px = await withTimeout(decodePixels(blob), 10000, 'thumbnail decode');
const t2 = performance.now();
const gray = toGray(px.rgba, px.w, px.h), fp = hashGray(gray, px.w, px.h);
// A small greyscale copy (at most 96 x 96 bytes) stays with the post for extended matching.
return { fp, small: grayCopy(gray, px.w, px.h), fetchMs: t1 - t0, decodeMs: t2 - t1, bitmapMs: px.bitmapMs, drawMs: px.drawMs,
hashMs: performance.now() - t2, bytes: blob.size };
}
// Videos and GIFs: matched like pictures, by their thumbnail (and exactly by MD5). The thumbnail of
// one is its first frame, which is often black or one flat colour: see thumbnailWeakness().
const isAnim = st => /^(webm|mp4|gif)$/.test(st.ext || '');
// ---------------------------------------------------------------- post discovery
const CLONE_CONTEXT = '#qp, .inline, .inlined, #quote-preview, .preview, .mbgx-panel';
const FILE_URL = /^(?:https?:)?\/\/(?:i\.4cdn\.org|is\d*\.4chan(?:nel)?\.org)\/(\w+)\/(\d+)\.(\w+)(?:[?#].*)?$/i;
const seenRoots = new WeakSet();
const rootState = new WeakMap(); // post container -> its per-post state
const posts = new Map(); // "board.no" -> per-post state; survives XT re-creating a post's element
/**
* 4chan thumbnails are always https://i.4cdn.org/<board>/<tim>s.jpg. Deriving the URL from the
* file link (instead of img.src) ignores spoiler placeholders, XT's "replace thumbnail with full
* image" options, expanded images and XT's full-image host rewriting.
*/
function thumbUrl(link, img) {
const m = FILE_URL.exec(link.getAttribute('href') || '');
if (m) return `https://i.4cdn.org/${m[1]}/${m[2]}s.jpg`;
return link.classList.contains('imgspoiler') || !img ? null : img.src;
}
function postFile(root) {
const post = root.querySelector(':scope > .post');
const file = post && post.querySelector(':scope > .file');
const link = file && file.querySelector('a.fileThumb'); // deleted files use span.fileThumb
if (!link) {
const thumb = root.matches('.catalog-container') && root.closest('.catalog-thread');
const src = thumb && /i\.4cdn\.org\/(\w+)\/(\d+)s\.jpg/.exec((thumb.querySelector('.catalog-thumb') || {}).src || '');
return src && post ? { post, link: null, img: null, url: `https://i.4cdn.org/${src[1]}/${src[2]}s.jpg` } : null;
}
const img = [...link.querySelectorAll('img')].find(i => !i.matches('.full-image, .expanded-thumb, .expanded-image'));
const url = thumbUrl(link, img), ext = FILE_URL.exec(link.getAttribute('href') || '');
return url && { post, link, img, url, ext: ext ? ext[3].toLowerCase() : null };
}
/**
* The file's MD5 from the page: 4chan and every X-family script put it on the thumbnail as data-md5
* (XT also on .fileInfo, which is all its catalog mode keeps). The native catalog page has none.
*/
function postMd5(f) {
const holder = f.post && f.post.querySelector(':scope > .file [data-md5]');
return normMd5((f.img && f.img.getAttribute('data-md5')) || (holder && holder.getAttribute('data-md5')));
}
const PAGE = /\/catalog(\/|$)/.test(location.pathname) || document.body.classList.contains('is_catalog') ? 'catalog'
: /\/(thread|res)\//.test(location.pathname) || document.body.classList.contains('is_thread') ? 'thread' : 'index';
const BOARD = location.pathname.split('/')[1] || '?';
/**
* What a matching image filters, following vanilla 4chan and 4chan XT (both only let filters hide
* a reply, or an OP on an index page):
* reply - that post only
* op - an OP on a thread page: never filtered (it can still be blacklisted)
* thread - an OP on an index page: the whole thread, with its shown replies
* tile - an OP in XT's catalog mode, or a native catalog tile: that catalog entry
*/
function roleOf(root) {
if (!root.matches('.opContainer, .catalog-container')) return { role: 'reply', target: root };
const tile = root.closest('.catalog-thread');
if (tile) return { role: 'tile', target: tile };
const thread = PAGE !== 'thread' && root.closest('.thread');
return thread ? { role: 'thread', target: thread } : { role: 'op', target: root };
}
/** Elements matching `selector` in `scope`, including `scope` itself. */
const within = (scope, selector) => scope === document ? scope.querySelectorAll(selector)
: scope.matches(selector) ? [scope] : scope.querySelectorAll(selector);
/** Find new posts and catalog tiles in `scope` (the document, or a node that was just added). */
function scan(scope = document) {
stats.scans++;
for (const root of within(scope, '.postContainer')) scanPost(root);
if (PAGE === 'catalog') for (const tile of within(scope, '#threads > .thread')) scanTile(tile);
}
function scanPost(root) {
if (seenRoots.has(root)) {
// XT moves an OP between index and catalog layouts without re-creating it: update its role.
const st = rootState.get(root);
if (st && st.root === root) {
const r = roleOf(root);
if (r.role !== st.role || r.target !== st.target) { unstub(st.target); Object.assign(st, r); applyMatch(st); }
}
return;
}
seenRoots.add(root);
// Originals have id="pc<digits>". XT clones get "_<n>" suffixes and live in #qp/.inline;
// vanilla inlines/previews are .inlined/.preview copies of .post.
const m = /^pc(\d+)$/.exec(root.id);
if (!m || root.closest(CLONE_CONTEXT)) { stats.clonesSkipped++; return; }
stats.posts++;
const f = postFile(root);
if (!f) { stats.noImage++; return; }
guard('attach', () => attach(root, m[1], f));
}
/**
* Thread number -> image ID for the native catalog, from the page's own inline `var catalog = {...}`
* data. It has the real image even when the tile only shows a spoiler. Read once per page.
*/
let catalogImages = null;
function catalogImage(no) {
if (!catalogImages) {
catalogImages = new Map();
const text = ([...document.scripts].find(x => /var catalog\s*=/.test(x.textContent)) || {}).textContent || '';
const start = text.indexOf('{', text.search(/var catalog\s*=/));
// The object ends at a "};", but a thread's text could contain one too: try the next if parsing fails.
for (let end = text.indexOf('};', start), tries = 0; start > 0 && end > 0 && tries < 20; end = text.indexOf('};', end + 2), tries++) {
try {
for (const [id, t] of Object.entries(JSON.parse(text.slice(start, end + 1)).threads || {})) {
if (/^\d+$/.test(t && t.imgurl)) catalogImages.set(id, t.imgurl);
}
break;
} catch (e) { /* not the end yet */ }
}
}
return catalogImages.get(no);
}
/** A native catalog tile: div.thread#thread-<no> > a > img.thumb. */
function scanTile(tile) {
if (seenRoots.has(tile)) return;
seenRoots.add(tile);
const m = /^thread-(\d+)$/.exec(tile.id);
if (!m) return;
stats.posts++;
const img = tile.querySelector('img.thumb'), src = img && /i\.4cdn\.org\/(\w+)\/(\d+)s\.jpg/.exec(img.getAttribute('src') || '');
const tim = catalogImage(m[1]) || (src && src[2]);
if (!tim) { stats.noImage++; return; }
guard('attach', () => attach(tile, m[1], { post: null, img, url: `https://i.4cdn.org/${BOARD}/${tim}s.jpg`, tile: true }));
}
function attach(root, no, f) {
const full = root.getAttribute('data-full-i-d'); // XT: dataset.fullID
const board = full ? full.split('.')[0] : BOARD;
const key = `${board}.${no}`;
let st = posts.get(key);
if (!st) {
const thread = f.tile ? null : root.closest('.thread');
st = { key, no, board, thread: f.tile ? no : thread ? thread.id.replace(/^t/, '') : null, url: f.url, ext: f.ext || null,
md5: postMd5(f), fp: null, promise: null, queued: false,
status: 'idle', error: null, result: null, revealed: false, busy: false, logged: {}, seenAt: performance.now() };
posts.set(key, st);
stats.imagePosts++;
}
st.root = root;
st.post = f.post;
Object.assign(st, f.tile ? { role: 'tile', target: root } : roleOf(root));
rootState.set(root, st);
if (root.classList.contains('post-hidden') || root.hidden || root.querySelector(':scope > .stub')) stats.alreadyHidden++;
if (!st.md5) st.md5 = postMd5(f); // XT can add it after the post first appears
if (st.post) addButton(st);
if (st.fp) stats.cacheHits++;
// Exact files are matched straight away from the page's MD5, before (and instead of) any download.
applyMatch(st);
if (!st.fp && ((autoHash() && !md5Matched(st)) || completes(st))) enqueue(st);
}
/** Background hashing: only needed with picture references, and not for posts already matched by MD5. */
const autoHash = () => cfg.enabled && activeSeeds().length > 0;
const md5Matched = st => !!(st.result && st.result.match && st.result.match.how === 'md5');
/** A post hidden by an exact-file reference that has no picture yet: its thumbnail completes the reference. */
const completes = st => cfg.completeMd5 && cfg.enabled && md5Matched(st) && !isPicture(st.result.match.seed) && !completeTried.has(st.result.match.seed.id);
// ---------------------------------------------------------------- hashing queue
// A thumbnail takes ~200 ms from 4chan's image server however many are in flight, so throughput
// scales with parallelism: 32 at once roughly halves a first visit to a big thread compared with 16,
// with request times unchanged. Repeat visits hit the browser cache.
const CONCURRENCY = 32;
const queue = [];
let active = 0, queueSorted = true, pumpScheduled = false, batch = null;
/** Background hashing gives up on a post after 2 failures; clicking its button still retries. */
function enqueue(st) {
if (st.fp || st.promise || st.queued || st.failures >= 2 || (md5Matched(st) && !completes(st))) return;
st.queued = true;
st.queuedAt = performance.now();
stats.queued++;
queue.push(st);
queueSorted = false;
renderButton(st);
// Start after the current scan has queued everything, so the queue can be sorted first.
if (!pumpScheduled) {
pumpScheduled = true;
queueMicrotask(() => { pumpScheduled = false; pump(); });
}
}
/** Posts nearest the visible part of the page first, so what you're looking at is filtered first. */
function sortQueue() {
const vh = innerHeight, dist = new Map();
for (const st of queue) {
const r = st.root.getBoundingClientRect(), d = !r.height ? 1e9 : r.bottom < 0 ? -r.bottom : r.top > vh ? r.top - vh : 0;
dist.set(st, d);
if (!d) st.inView = true; // for diagnostics: in view when queued
}
queue.sort((x, y) => dist.get(x) - dist.get(y));
queueSorted = true;
}
function pump() {
if (!queueSorted) sortQueue();
if (!batch && queue.length) { batch = { start: performance.now(), n: 0 }; watchStalls(); }
while (active < CONCURRENCY && queue.length) {
const st = queue.shift();
st.queued = false;
if (st.queuedAt) addTiming(stats.queueMs, performance.now() - st.queuedAt);
active++;
batch.n++;
ensureHash(st).catch(() => {}).finally(() => {
active--;
if (!active && !queue.length && batch) {
const end = performance.now();
// fromStart: since the script started (the whole initial wait, scanning and queueing included)
stats.lastBatch = { thumbnails: batch.n, seconds: Math.round(end - batch.start) / 1000, fromStart: Math.round(end - scriptStart) / 1000 };
if (!stats.firstBatch) stats.firstBatch = stats.lastBatch;
batch = null;
log('queue-drained', { ...stats.lastBatch, hashOK: stats.hashOK, hashErrors: stats.hashErrors, matched: stats.matched, avgHashMs: stats.hashMs.n ? round(stats.hashMs.total / stats.hashMs.n) : null, avgFetchMs: stats.fetchMs.n ? round(stats.fetchMs.total / stats.fetchMs.n) : null });
updateControl();
}
pump();
});
}
}
const batchText = b => `${b.thumbnails} thumbnail${b.thumbnails === 1 ? '' : 's'} in ${b.seconds.toFixed(2)} s` + (b.fromStart != null ? ` (done ${b.fromStart.toFixed(2)} s after start)` : '');
/** While a batch runs: how late a 50 ms timer fires shows how long the page was busy (any script, GC, layout). */
function watchStalls() {
let last = performance.now();
const tick = () => {
const now = performance.now(), late = now - last - 50;
if (late > 50) addTiming(stats.stalls, late);
last = now;
if (batch) setTimeout(tick, 50);
};
setTimeout(tick, 50);
}
function startBackgroundHashing() {
if (!autoHash()) return;
for (const st of posts.values()) if (st.root && st.root.isConnected) enqueue(st);
}
/** Single in-flight hash per post; a failure clears the promise so the next attempt really retries. */
function ensureHash(st) {
if (st.fp) return Promise.resolve(st.fp);
if (!st.promise) {
st.status = 'hashing';
renderButton(st);
st.promise = fingerprint(st.url).then(r => {
st.fp = r.fp;
st.px = r.small;
st.status = 'ok';
stats.hashOK++;
addTiming(stats.fetchMs, r.fetchMs);
addTiming(stats.decodeMs, r.decodeMs);
addTiming(stats.bitmapMs, r.bitmapMs);
addTiming(stats.drawMs, r.drawMs);
addTiming(stats.hashMs, r.hashMs);
stats.thumbBytes += r.bytes;
if (cfg.verbose) log('hashed', { post: st.key, url: st.url, a: r.fp.a, d: r.fp.d, w: r.fp.w, h: r.fp.h, low: r.fp.low, fetchMs: round(r.fetchMs), hashMs: round(r.hashMs), bytes: r.bytes });
guard('applyMatch', () => applyMatch(st));
return r.fp;
}, e => {
st.status = 'error';
st.error = String(e && e.message || e);
st.promise = null;
st.failures = (st.failures || 0) + 1;
stats.hashErrors++;
log('hash-error', { post: st.key, url: st.url, error: st.error });
throw e;
}).finally(() => renderButton(st));
}
return st.promise;
}
// ---------------------------------------------------------------- matching and stubs
/**
* "Measure matching speed": the same checks as applyMatch() on every hashed post of the page,
* changing nothing, 25 posts at a time between pauses. While a page loads, its matching times include
* waits for other work (other scripts, garbage collection); these show the script's own speed.
* Each group of 25 is timed as a whole: browsers may round the clock to a millisecond, too coarse for
* one picture. Ordinary matching first, then edited copies for the pictures it didn't match.
* done(text) gets a summary, also logged as "matching-timed".
*/
function measureMatching(done) {
const list = [...posts.values()].filter(st => st.fp && st.px && !st.fp.low), ext = extendedWays(), rest = [];
const ordinary = { n: 0, ms: 0, worst: 0 }, edited = { n: 0, ms: 0, worst: 0 };
const groups = (items, check, sum, next) => {
let i = 0;
const step = () => {
const end = Math.min(items.length, i + 25), t0 = performance.now(), n = end - i;
for (; i < end; i++) check(items[i]);
const ms = performance.now() - t0;
sum.n += n; sum.ms += ms; sum.worst = Math.max(sum.worst, n ? ms / n : 0);
if (i < items.length) setTimeout(step, 0); else next();
};
step();
};
const per = x => (x.n ? Math.round(x.ms / x.n * 1000) / 1000 : null);
groups(list, st => {
const r = matchPrepared(st.fp, refDB, allowSet, cfg, st.board), c = ordinaryDetail(st.px, refDB, r.match);
if (!((r.match && !(c !== null && c < ORD_DETAIL)) || r.allowed) && (ext || refDB.anyClose)) rest.push(st);
}, ordinary, () => groups(rest, st => {
const look = ext && matchLookalike(st.px, st.fp, refDB, cfg, st.board, LOOK);
if (!look && ext && !matchExtended(st.px, refDB, cfg, st.board, ext) && ext.overlap) matchOverlap(st.px, refDB, cfg, st.board);
if (!look && refDB.anyClose) matchLookalike(st.px, st.fp, refDB, cfg, st.board, CLOSE);
}, edited, () => {
const res = { thumbnails: list.length, references: refDB.seeds.length, ordinaryMs: per(ordinary), editedMs: per(edited), editedChecked: edited.n,
editedWorstGroupMs: Math.round(edited.worst * 1000) / 1000, whileLoadingMs: stats.extMs.n ? Math.round(stats.extMs.total / stats.extMs.n * 1000) / 1000 : null };
log('matching-timed', res);
done(list.length ? `${list.length} thumbnails, ${res.references} references: ordinary ${res.ordinaryMs} ms, edited copies ${res.editedMs} ms per picture ` +
`(${res.editedChecked} checked); while loading ${res.whileLoadingMs} ms.` : 'No hashed thumbnails on this page yet.');
}));
}
function applyMatch(st) {
const before = st.result && st.result.match;
// 1. The exact file, by the MD5 4chan gives (valid however blank the thumbnail is).
const exact = st.md5 && refDB.md5Seeds.length ? matchMd5(st.md5, refDB, allowSet, st.board) : null; // per category (prepareReferences)
if (!exact && !st.fp) {
if (st.result && st.result.match) { st.result = null; unstub(st.target); renderButton(st); updateControl(); }
return;
}
const t0 = performance.now();
const r = st.result = exact ? { match: { seed: exact, da: null, dd: null, how: 'md5' }, best: null, near: false, allowed: false, low: !!(st.fp && st.fp.low) }
: matchPrepared(st.fp, refDB, allowSet, cfg, st.board);
// 2. The picture (ordinary matching above; near the limits it must also look like the reference's detail)...
const corr = exact ? null : ordinaryDetail(st.px, refDB, r.match);
if (corr !== null && corr < ORD_DETAIL) {
r.detailVeto = Math.round(corr * 1000) / 1000;
r.match = null;
r.near = !r.allowed && !r.low;
}
addTiming(stats.matchMs, performance.now() - t0);
// 3. ...then retouched copies (drawn over, darkened, recoloured), edited copies (cropped, bordered,
// mirrored, inverted, in a collage) and, only if turned on, close variants.
if (!r.match && !r.allowed && !r.low && st.px && (extendedWays() || refDB.anyClose)) {
// to a worker; while the workers are still preparing, a post that was in view is checked on the page instead
if (pool.state === 'on' && (pool.live || !st.inView)) {
// finished when the worker answers; meanwhile a hidden post stays hidden, unless its reference is gone
if (before && !refApplies(before.seed.id, st.board)) { unstub(st.target); renderButton(st); updateControl(); }
sendToWorkers(st, r, extendedWays());
return;
}
matchOnPage(st, r);
return;
}
finishMatch(st, r);
}
/** The edited-copy checks on the page itself (no workers), then the rest of applyMatch(). */
function matchOnPage(st, r) {
const ext = extendedWays();
const t1 = performance.now(), corr3 = c => (c == null ? null : Math.round(c * 1000) / 1000);
const look = ext ? matchLookalike(st.px, st.fp, refDB, cfg, st.board, LOOK) : null;
addTiming(stats.lookMs, performance.now() - t1);
const count = { windows: 0, aligns: 0, evals: 0 };
const x = !look && ext ? matchExtended(st.px, refDB, cfg, st.board, ext, count) || (ext.overlap ? matchOverlap(st.px, refDB, cfg, st.board, count) : null) : null;
if (!look && !x && ext && ext.overlap) st.ovPending = !overlapReady(refDB); // checked again once the index is built
stats.extWindows += count.windows;
stats.extAligns += count.aligns;
const close = !look && !x && refDB.anyClose ? matchLookalike(st.px, st.fp, refDB, cfg, st.board, CLOSE) : null;
addTiming(stats.extMs, performance.now() - t1);
if (look || close) {
const m = look || close;
r.match = { seed: m.seed, da: m.da, dd: m.dd, rule: m.rule, how: look ? 'retouch' : 'close', corr: corr3(m.corr) };
r.near = false;
} else if (x) {
r.match = { seed: x.seed, da: round(x.a64), dd: round(x.d64), rule: x.rule, how: x.how, corr: corr3(x.corr) };
r.near = false;
}
finishMatch(st, r);
}
/** After matching: report and record the match (or near miss), stub or unstub the post. */
function finishMatch(st, r) {
// A match of a reference's main picture (its largest panel) is reported as a match of the reference.
if (r.match && r.match.seed.isPart && !r.match.how) r.match.how = r.match.seed.isPiece ? 'piece' : 'part';
if (r.match && st.fp) fillDetail(st, r.match);
if (r.match && r.match.how === 'md5' && st.fp && st.px && completes(st)) completeFromPost(st, r.match.seed);
const info = { post: st.key, url: st.url, ...(r.best ? { seed: r.best.seed.id, da: r.best.da, dd: r.best.dd } : {}), ...(st.fp ? { a: st.fp.a, d: st.fp.d } : {}), ...(st.md5 ? { md5: st.md5 } : {}) };
if (r.low && !st.logged.low) {
st.logged.low = true;
stats.lowDetail++;
if (cfg.verbose) log('low-detail-skip', { post: st.key, url: st.url, std: st.fp && st.fp.std });
}
if (r.match && st.role === 'op' && !st.logged.op) {
st.logged.op = true;
log('op-not-filtered', { post: st.key, seed: r.match.seed.id, da: r.match.da, dd: r.match.dd });
}
if (r.match && cfg.enabled && st.role !== 'op') {
if (st.logged.match !== r.match.seed.id) {
st.logged.match = r.match.seed.id;
stats.matched++;
const how = r.match.how || 'hash';
stats.byMethod[how] = (stats.byMethod[how] || 0) + 1;
log('matched', { ...info, seed: r.match.seed.id, how, da: r.match.da, dd: r.match.dd, ...(r.match.corr != null ? { corr: r.match.corr } : {}), revealed: st.revealed });
}
if (!st.revealed) stub(st);
xAutoAdd(st);
} else {
unstub(st.target);
if (r.near && !st.logged.near) {
st.logged.near = true;
stats.nearMisses++;
log('near-miss', { ...info, allowed: r.allowed });
}
}
if (st.fp && st.seenAt && !st.decidedAt) {
st.decidedAt = performance.now();
addTiming(stats.decisionMs, st.decidedAt - st.seenAt);
if (st.inView) addTiming(stats.visibleDecisionMs, st.decidedAt - st.seenAt);
if (stats.hashOK % 25 === 1) sampleHeap();
}
renderButton(st);
updateControl();
}
/** Ways of extended matching that are on (null when none): see matchExtended(). */
/** The edited-copy checks, if any reference has them on (generally, or through its category's own rules). */
function extendedWays() {
return refDB.anyExt ? { crop: true, border: true, mirror: true, panels: true, invert: true, zoom: true, tilt: true, turn: true, overlap: true } : null;
}
/** How a match was made, in words; distances of extended matches are per 64 compared bits. */
const METHOD = { md5: 'exact file (MD5)', hash: 'same picture', crop: 'cropped or reframed copy', border: 'copy with added borders',
mirror: 'mirror image', flip: 'upside-down copy', zoom: 'cropped or reframed copy', tilt: 'tilted copy', turn: 'rotated copy', overlap: 'overlapping crop', panel: 'part of a collage', invert: 'inverted copy',
retouch: 'retouched copy (drawn over or recoloured)', close: 'close variant', part: 'its main picture', piece: 'heavily cropped copy' };
const methodOf = m => METHOD[m.how || 'hash'] + (m.seed && m.seed.isPiece ? (m.how !== 'piece' ? ' of part of it' : '')
: m.seed && m.seed.isPart && m.how !== 'part' ? ' of its main picture' : '') +
(m.corr != null ? ` (detail ${Math.round(m.corr * 100)}%)` : '');
/**
* A reference without a detail gets one when its exact picture (same hashes) is seen again, so
* later edited copies of it are checked against the detail as well; likewise its main picture
* ("part") if it is a collage or has an added band, and its pieces.
*/
const pendingDetails = new Map(), partChecked = new Set();
function fillDetail(st, m) {
const s = m.seed;
if (m.how || s.isPart || !st.px || !st.fp || s.a !== st.fp.a || s.d !== st.fp.d) return;
const add = {};
if (!s.detail) add.detail = detailOf(integral(st.px));
if (!partChecked.has(s.id)) {
partChecked.add(s.id);
const part = !s.part && pictureOf(st.px), pieces = !s.pieces && piecesOf(st.px, st.fp);
if (part) add.part = part;
if (pieces && pieces.length) add.pieces = packPieces(pieces);
}
if (!Object.keys(add).length) return;
Object.assign(s, add);
if (add.detail) {
const i = refDB.seeds.indexOf(s);
refDB.details.delete(i);
for (const entry of pool.workers) entry.w.postMessage({ type: 'detail', version: pool.version, i, detail: add.detail });
}
pendingDetails.set(s.id, { ...(pendingDetails.get(s.id) || {}), ...add });
if (pendingDetails.size === 1) setTimeout(() => guard('details', () => {
const list = loadSeeds();
let n = 0, added = false; // main pictures and pieces are new entries of the reference database
for (const x of list) {
const a = pendingDetails.get(x.id);
if (!a) continue;
if (a.detail && !x.detail) { x.detail = a.detail; n++; }
if (a.part && !x.part) { x.part = a.part; n++; added = true; }
if (a.pieces && !x.pieces) { x.pieces = a.pieces; n++; added = true; }
}
pendingDetails.clear();
if (n) { write('seeds', list); log('details-added', { fields: n }); if (added) prepareData(); }
}), 3000);
}
function rematchAll() {
seeds = loadSeeds();
allow = loadAllow();
cats = loadCats();
prepareData();
for (const st of posts.values()) guard('rematch', () => applyMatch(st));
applyStubMode(); // a category's stub choice may have changed for posts that stay filtered
startBackgroundHashing();
updateControl();
}
function el(tag, props = {}, ...kids) {
const e = document.createElement(tag);
for (const [k, v] of Object.entries(props)) {
if (k === 'dataset') Object.assign(e.dataset, v);
else if (k.startsWith('on')) e.addEventListener(k.slice(2), v);
else if (k in e) e[k] = v;
else e.setAttribute(k, v);
}
e.append(...kids.flat(Infinity).filter(x => x != null && x !== false));
return e;
}
/**
* In-page replacements for confirm()/alert(). Firefox lets users block a site's dialogs ("Don't
* allow boards.4chan.org to prompt you again"); confirm() then silently returns false and alert()
* shows nothing, which would make every confirmation here fail. Resolves true for OK, false for
* Cancel, Escape or a click outside. A blank line in the message starts a new paragraph.
*/
function ask(message, okLabel = 'OK', cancelLabel = 'Cancel', fields = null) {
return new Promise(resolve => {
const done = value => {
removeEventListener('keydown', onKey, true);
back.remove();
box.remove();
resolve(value);
};
// Window-level capture runs before the panel's own Escape handler, so only this dialog closes.
const onKey = e => {
if (e.key !== 'Escape' && e.key !== 'Enter') return;
if (e.key === 'Enter' && e.target.matches && e.target.matches('select, textarea')) return;
e.preventDefault();
e.stopPropagation();
done(e.key === 'Enter');
};
const back = el('div', { className: 'mbgx-backdrop mbgx-dialog-backdrop', onclick: () => done(false) });
const ok = el('button', { type: 'button', className: 'mbgx-dialog-ok', onclick: () => done(true) }, okLabel);
guard('control', applyTheme);
const box = el('div', { className: 'mbgx-panel mbgx-dialog', role: 'dialog' },
el('h3', {}, 'Better Filter'),
String(message).split('\n\n').map(t => el('p', {}, t)),
fields,
el('p', {}, ok, cancelLabel && el('button', { type: 'button', className: 'mbgx-dialog-cancel', onclick: () => done(false) }, cancelLabel)));
document.body.append(back, box);
addEventListener('keydown', onKey, true);
(box.querySelector('input[type=text], input[type=number]') || ok).focus();
});
}
const notify = message => ask(message, 'OK', null);
const cmd = (key, name, label, title) => el('a', { className: 'mbgx-cmd', role: 'button', tabIndex: 0, title, dataset: { mbgxKey: key, mbgxCmd: name } }, label);
/** Stub is a sibling placed before .post inside the post container; .post itself is only hidden by CSS. */
/**
* Stub inside the filtered element: before .post for a reply (only .post is hidden), first child of
* a thread or catalog tile (everything else in it is hidden). The original nodes are never removed.
*/
function stub(st) {
const target = st.target, m = st.result && st.result.match;
if (!target || !m) return;
let box = target.querySelector(':scope > .mbgx-stub');
const fresh = !box;
if (fresh) box = el('div', { className: 'mbgx-stub' });
const detail = m.how === 'md5' ? `reference ${m.seed.id} · ${methodOf(m)}`
: m.how ? `reference ${m.seed.id} · ${methodOf(m)} · aHash ${m.da}/64 · dHash ${m.dd}/64`
: `reference ${m.seed.id} · aHash ${m.da}/64 (≤${m.rule.a}) · dHash ${m.dd}/64 (≤${m.rule.d})`;
const reveal = cmd(st.key, 'reveal', 'Reveal', 'Show it again (until the page is reloaded)');
const whitelist = cmd(st.key, 'whitelist', 'False positive', 'Never filter this exact image again; the reference stays active');
if (st.role === 'tile') {
box.title = `Better Filter filtered thread No.${st.no}: ${detail}`;
box.replaceChildren(el('span', { className: 'mbgx-stub-label' }, `⊘ filtered (${m.seed.id})`), el('br'), reveal, whitelist);
} else {
const what = st.role === 'thread' ? 'thread' : 'post';
box.replaceChildren(
el('span', { className: 'mbgx-stub-label' }, `⊘ Better Filter filtered ${what} No.${st.no} — ${detail}`),
st.role === 'thread' ? el('a', { className: 'mbgx-cmd', href: `/${st.board}/thread/${st.no}`, title: 'Open the thread' }, 'Open thread') : '',
reveal,
cmd(st.key, 'whitelist', 'False positive / whitelist', 'Never filter this exact image again; the reference stays active'),
cmd(st.key, 'unseed', `Remove reference ${m.seed.id}`, 'Delete this reference image from the blacklist (undo a mistaken blacklist)'));
}
if (fresh) {
if (st.role === 'reply') target.insertBefore(box, st.post);
else target.prepend(box);
stats.stubbed++;
nudgeUnread();
}
target.classList.remove('mbgx-by-reply');
target.classList.add('mbgx-filtered');
target.classList.toggle('mbgx-nostub', !stubsFor(m));
if (nativeHidden(target)) watchNative(st);
scheduleQuotes();
}
/** Hidden by 4chan (class post-hidden) or 4chan XT (its .stub, or the hidden attribute) on its own. */
const nativeHidden = e => e.hidden || e.classList.contains('post-hidden') || !!e.querySelector(':scope > .stub');
const nativeName = () => (document.documentElement.classList.contains('fourchan-x') ? '4chan X' : '4chan');
/**
* A post 4chan or 4chan XT already hides keeps their stub (CSS hides ours). If you show it there,
* that is an explicit choice: Better Filter treats it as revealed too instead of keeping it hidden.
*/
function watchNative(st) {
if (st.nativeWatch) return;
const t = st.target;
st.nativeWatch = new MutationObserver(() => {
if (nativeHidden(t)) return;
st.nativeWatch.disconnect();
st.nativeWatch = null;
if (st.result && st.result.match && !st.revealed && st.target === t && t.classList.contains('mbgx-filtered')) {
log('native-revealed', { post: st.key });
revealPost(st);
}
});
st.nativeWatch.observe(t, { childList: true, attributes: true, attributeFilter: ['hidden', 'class'] });
}
/**
* Unread counts, e.g. "(1)" in the tab title. Vanilla 4chan counts every new post, filtered or not,
* so nothing is needed there. 4chan XT leaves out posts its own filters hide, but it decides when a
* post arrives, before its thumbnail can be checked here. XT drops a hidden post from its count when
* it next checks what has been read, which it does on scroll: so after filtering, prompt that check.
* (XT only checks while the tab is visible, and treats a post whose stub is shown like any visible post.)
*/
let unreadNudge = 0;
function nudgeUnread() {
if (unreadNudge || !document.documentElement.classList.contains('fourchan-x')) return;
unreadNudge = setTimeout(() => {
unreadNudge = 0;
document.dispatchEvent(new Event('scroll'));
}, 150);
}
function unstub(root) {
if (!root || root.classList.contains('mbgx-by-reply')) return; // hidden as a reply: updateQuotes() decides
const box = root.querySelector(':scope > .mbgx-stub');
if (box) box.remove();
root.classList.remove('mbgx-filtered', 'mbgx-nostub');
scheduleQuotes();
}
/**
* Links to and replies to filtered posts, following 4chan XT (vanilla 4chan leaves both alone):
* - quote links and backlinks to a filtered post are struck through; backlinks are hidden instead
* when XT is set to hide them (its "Filtered Backlinks" setting off adds :root.hide-backlinks);
* - with "Also hide replies to filtered posts" on (off by default, as XT's filters), a reply to a
* filtered post is hidden too, recursively, like XT's per-filter "recursive" option. Revealing
* the filtered post shows its replies again, as in XT; each can also be revealed on its own.
* Hover previews and inline quotes of a filtered post show it in full, as both do (see CSS).
*/
let quotesTimer = 0;
const revealedReplies = new Set();
function scheduleQuotes() {
if (!quotesTimer) quotesTimer = setTimeout(() => { quotesTimer = 0; guard('quotes', updateQuotes); }, 60);
}
const quoteTarget = a => { const m = /#p(\d+)$/.exec(a.getAttribute('href') || ''); return m && m[1]; };
function updateQuotes() {
const hidden = new Map(); // post number -> the filtered post it leads back to
// filtered post -> whether replies to it are hidden, and with a stub (its reference's category, else the general settings)
const hides = new Map(), stubbed = new Map();
for (const st of posts.values()) {
if (st.role === 'reply' && st.root && st.root.isConnected && st.root.classList.contains('mbgx-filtered') && !st.root.classList.contains('mbgx-by-reply')) {
const cat = st.result && st.result.match && st.result.match.seed.cat;
hidden.set(st.no, st.no);
hides.set(st.no, catSetting(cat, 'hideReplies'));
stubbed.set(st.no, catSetting(cat, 'stubs'));
}
}
for (const root of document.querySelectorAll('.replyContainer')) {
const m = /^pc(\d+)$/.exec(root.id);
if (!m || root.closest(CLONE_CONTEXT) || hidden.has(m[1])) continue;
const st = rootState.get(root);
const src = cfg.enabled && !revealedReplies.has(m[1]) && !(st && st.revealed) &&
[...root.querySelectorAll(':scope > .post > .postMessage a.quotelink')].map(quoteTarget).find(t => t && t !== m[1] && hidden.has(t) && hides.get(hidden.get(t)));
if (src) {
hidden.set(m[1], hidden.get(src));
replyStub(root, m[1], src, !stubbed.get(hidden.get(src)));
} else if (root.classList.contains('mbgx-by-reply')) {
root.classList.remove('mbgx-by-reply');
unstub(root);
}
}
for (const a of document.querySelectorAll('a.quotelink, a.backlink')) {
const t = quoteTarget(a);
a.classList.toggle('mbgx-quote-filtered', !!t && hidden.has(t));
}
updateControl();
}
function replyStub(root, no, src, noStub) {
root.classList.toggle('mbgx-nostub', noStub); // as the filtered post it leads back to
if (root.classList.contains('mbgx-by-reply') && root.dataset.mbgxReplyTo === src) return;
let box = root.querySelector(':scope > .mbgx-stub');
if (!box) root.insertBefore(box = el('div', { className: 'mbgx-stub' }), root.querySelector(':scope > .post'));
box.replaceChildren(el('span', { className: 'mbgx-stub-label' }, `⊘ Better Filter hid No.${no}: a reply to filtered post No.${src}`),
el('a', { className: 'mbgx-cmd', role: 'button', tabIndex: 0, title: 'Show this reply', dataset: { mbgxCmd: 'reveal-reply', mbgxNo: no } }, 'Reveal'));
root.dataset.mbgxReplyTo = src;
root.classList.add('mbgx-filtered', 'mbgx-by-reply');
}
/**
* The button goes at the end of the file-info line (after XT's google/yandex/... links), where
* image actions live.
*/
function addButton(st) {
const file = st.post.querySelector(':scope > .file');
const host = file && (file.querySelector(':scope > .fileText') || file);
if (!host) return;
let b = host.querySelector(':scope > .mbgx-action');
if (!b) {
b = el('a', { className: 'mbgx-action', role: 'button', tabIndex: 0, dataset: { mbgxKey: st.key } });
if (host === file) file.prepend(b);
else host.append(' ', b);
stats.buttons++;
}
st.buttons = [b];
// 4chan's mobile layout hides the file-info line and shows the .postInfoM header instead: a second
// button there carries 4chan's own "mobile" class, so its stylesheet shows exactly one of the two.
const mobileInfo = st.post.querySelector(':scope > .postInfoM');
if (mobileInfo) {
let mb = mobileInfo.querySelector(':scope > .mbgx-action');
if (!mb) mobileInfo.append(mb = el('a', { className: 'mbgx-action mobile', role: 'button', tabIndex: 0, dataset: { mbgxKey: st.key } }));
st.buttons.push(mb);
}
renderButton(st);
}
/**
* Touch screens (no hover) can't show title tooltips: in Better Filter's own controls, a long
* press (not on a link or picture), or a tap on something that isn't a control (a tag, a table cell), shows the title
* in a small popup instead. It goes away on the next touch, on scrolling, or after 8 s.
*/
const OWN_UI = '.mbgx-panel, .mbgx-stub, .mbgx-action, .mbgx-dialog, .mbgx-cmd';
function touchTips() {
let tip = null, timer = 0;
const hide = () => { clearTimeout(timer); if (tip) { tip.remove(); tip = null; } };
const show = (t, x, y) => {
hide();
tip = el('div', { className: 'mbgx-touch-tip', role: 'tooltip' }, t.title);
document.body.append(tip);
const r = tip.getBoundingClientRect();
tip.style.left = `${Math.max(8, Math.min(innerWidth - r.width - 8, x - r.width / 2))}px`;
tip.style.top = `${y + 24 + r.height < innerHeight ? y + 24 : Math.max(8, y - r.height - 12)}px`;
timer = setTimeout(hide, 8000);
};
const noHover = () => matchMedia('(hover: none)').matches;
const owner = e => { const t = e.target.closest && e.target.closest('[title]'); return t && t.title && t.closest(OWN_UI) ? t : null; };
document.addEventListener('contextmenu', e => {
const t = noHover() && owner(e);
if (!t || e.target.closest('a[href], img')) return; // links and pictures keep the browser's own menu
e.preventDefault();
show(t, e.clientX, e.clientY);
}, true);
document.addEventListener('click', e => {
const t = noHover() && owner(e);
if (!t || e.target.closest('a, button, input, select, textarea, summary, label')) return;
show(t, e.clientX, e.clientY);
}, true);
document.addEventListener('touchstart', e => { if (tip && !tip.contains(e.target)) hide(); }, { capture: true, passive: true });
addEventListener('scroll', hide, { passive: true });
}
/** Move a file line's "⊘ blacklist" (and the space before it) back to the end if something was added after it. */
function keepButtonLast(host) {
const b = host.querySelector(':scope > .mbgx-action');
if (!b || !b.nextSibling) return;
const sp = b.previousSibling;
if (sp && sp.nodeType === 3 && sp.textContent === ' ') sp.remove();
host.append(' ', b);
}
function renderButton(st) {
if (!st.buttons) return;
const r = st.result;
let text = '⊘ blacklist', title = 'Better Filter: blacklist this image (hide posts with matching images)';
if (st.busy || st.queued || st.status === 'hashing') text += ' …';
else if (st.status === 'error') {
text += ' (retry)';
title = `Thumbnail hashing failed: ${st.error}\nClick to retry.`;
}
if (r && r.match && st.revealed) {
text = `⊘ hide again (${r.match.seed.id})`;
title = 'This post matches a reference and was revealed. Click to hide it again.';
}
if (r && r.match && r.match.how === 'md5' && !st.revealed) title += `\n\nThis exact file is reference ${r.match.seed.id} (MD5).`;
if (st.fp) {
title += `\n\naHash ${st.fp.a} dHash ${st.fp.d} (${st.fp.w}×${st.fp.h})`;
if (r && r.best) title += `\nClosest reference ${r.best.seed.id}: aHash ${r.best.da}/64, dHash ${r.best.dd}/64`;
if (r && r.allowed) title += '\nWhitelisted as a false positive.';
if (r && r.low) title += '\nNearly blank thumbnail: filtered by picture only by a reference you made from a nearly blank thumbnail (its exact file always can be).';
}
if (r && r.match && st.role === 'op') title += `\nMatches ${r.match.seed.id}, but OPs are never filtered on thread pages (as in 4chan and 4chan XT).`;
for (const b of st.buttons) {
b.textContent = text;
b.title = title;
}
}
// ---------------------------------------------------------------- references, categories and tags
function nextSeedId(list = seeds) {
let max = 0; // a loop, not Math.max(...): a spread of hundreds of thousands of numbers can overflow the stack
for (const s of list) { const m = /^r(\d+)$/.exec(s.id); if (m && +m[1] > max) max = +m[1]; }
return 'r' + (max + 1);
}
function nextCatId(list = cats) {
const n = list.map(c => /^c(\d+)$/.exec(c.id)).filter(Boolean).map(m => +m[1]);
return 'c' + (n.length ? Math.max(...n) + 1 : 1);
}
/** A board list as shown and typed: comma-separated, e.g. "v, vg, a". */
const boardsText = list => (list || []).join(', ');
/**
* Parse a board list: separated by commas, spaces ignored, slashes optional, so "v, vg, a",
* "v,vg,a" and "/v/,/vg/" all work. Returns null (after saying why) if a name is not a board name.
*/
function parseBoards(text) {
const list = [...new Set(String(text).toLowerCase().replace(/\s+/g, '').split(',').map(t => t.replace(/^\/|\/$/g, '')).filter(Boolean))];
const bad = list.filter(b => !/^[a-z0-9]{1,12}$/.test(b));
if (bad.length) { notify(`Not a board name: ${bad.join(', ')}\n\nSeparate boards with commas, e.g. "v, vg, a", or leave the box empty for all boards.`); return null; }
return list;
}
/** Tags: comma-separated, trimmed, at most 20 of up to 40 characters. */
/** "1 reference", "2 references". */
const refCount = n => `${n} reference${n === 1 ? '' : 's'}`;
const parseTags = text => [...new Set(String(text).split(',').map(t => t.trim().slice(0, TAG_CHARS)).filter(Boolean))].slice(0, TAG_LIMIT);
/** Tags that would go past the limit: how many were not kept. */
const tagsOver = list => Math.max(0, new Set(list).size - TAG_LIMIT);
const catById = id => cats.find(c => c.id === id) || null;
const ruleText = r => `aHash ≤ ${r.a}, dHash ≤ ${r.d}${r.mode === 'either' ? ', either hash' : ''}, near-miss margin ${r.margin}`;
/** A category's own settings in a few words, e.g. "stubs off, edited copies on". */
const SETTING_WORDS = { stubs: 'stubs', hideReplies: 'hiding replies', xAdd: 'adding to 4chan X', xAuto: 'auto-adding to 4chan X',
xNoStub: '4chan X lines without stubs', md5: 'exact files', edited: 'edited copies', pieces: 'heavily cropped copies', close: 'close variants' };
const settingsText = set => Object.entries(set || {}).map(([k, v]) => `${SETTING_WORDS[k] || k} ${v ? 'on' : 'off'}`).join(', ');
const scopeText = list => (list && list.length ? list.map(b => `/${b}/`).join(' ') : 'all boards');
/** Where a reference applies and by which rules, with the reason, for the panel. */
function effective(s) {
const c = catById(s.cat);
return {
boards: s.boards && s.boards.length ? s.boards : c && c.boards ? c.boards : null,
boardsFrom: s.boards && s.boards.length ? 'own' : c && c.boards ? 'category' : 'none',
rules: c && c.rules ? c.rules : cfg, rulesFrom: c && c.rules ? 'category' : 'general',
active: !c || c.enabled !== false,
};
}
/** Change many references with one storage write and one re-check of the page. */
function editSeeds(fn, what) {
write('seeds', fn(loadSeeds()));
if (what) log('references-edited', what);
rematchAll();
flush();
}
function saveCats(list, what) {
write('categories', list.map(cleanCat));
if (what) log('categories-edited', what);
rematchAll();
flush();
}
/** Category with this name (any case), created if needed. Returns [category, updated list]. */
function categoryNamed(name, list = loadCats()) {
const clean = String(name).trim().slice(0, 60) || 'New category';
const found = list.find(c => c.name.toLowerCase() === clean.toLowerCase());
if (found) return [found, list];
const c = cleanCat({ id: nextCatId(list), name: clean, enabled: true });
return [c, [...list, c]];
}
/** Category <select> for dialogs: '' = Uncategorised, 'new' = create one. */
function categorySelect(className, selected, withNew = true) {
return el('select', { className },
el('option', { value: '', selected: !selected }, 'Uncategorised'),
cats.map(c => el('option', { value: c.id, selected: c.id === selected }, c.name + (c.enabled === false ? ' (disabled)' : ''))),
withNew && el('option', { value: 'new' }, '+ New category…'));
}
// ---------------------------------------------------------------- 4chan X family: their MD5 filter
//
// 4chan X (ccd0) and its forks 4chan XT (and the burningwiththefiresoforc fork), 4chan XZ and
// 4chan neXT keep MD5 filter rules in their own userscript storage, which no other script can read
// or write: one rule per line, "/<base64 MD5>/" plus options such as ";boards:vg,a;stub:no;".
// None offers an event to add a rule. Better Filter only ever adds lines, never edits or removes
// one, and never synchronises: removing a Better Filter reference leaves their rule alone.
/**
* Which X-family script runs on this page, from what it marks <html> with: every fork adds
* "fourchan-x"; XT, XZ and neXT also "fourchan-xt"; neXT also "fourchan-next". Only one fork runs
* per page (each stops if another already started); "tainted" means two were enabled anyway.
* XT and XZ mark the page identically, so the exact name and version are read from their settings
* window's credits the first time it is opened from here.
*/
let xIdentity = null;
function xFamily() {
const cl = document.documentElement.classList;
if (!cl.contains('fourchan-x')) return null;
const kind = cl.contains('fourchan-next') ? 'neXT' : cl.contains('fourchan-xt') ? 'XT' : 'X';
const guess = { neXT: '4chan neXT', XT: '4chan XT or XZ', X: '4chan X' }[kind];
return { kind, name: xIdentity ? xIdentity.name : guess, version: xIdentity && xIdentity.version, tainted: cl.contains('tainted'),
shiftClick: kind !== 'X' }; // XT, XZ and neXT have "MD5 Quick Filter" (Shift+click a thumbnail); 4chan X has not
}
/**
* Open the X-family settings window on its MD5 filter, run `work(textarea, info)`, close it again.
* This is what you would do by hand: their documented "OpenSettings" event opens the Filter
* section, "Image MD5" is chosen, and a "change" on the textarea is what saves it in every fork
* (XZ's rule editor keeps the same textarea under "Raw rules"; neXT's under "Advanced"). The window
* is kept invisible meanwhile. Refuses when their settings are already open (yours to finish).
*/
let xBusy = null;
function withXFilter(work) {
const fam = xFamily();
if (!fam) return Promise.reject(new Error('no 4chan X-family script is running on this page'));
if (fam.tainted) return Promise.reject(new Error(`${fam.name}: two copies are enabled (it says so itself); not touching its filters`));
if (document.getElementById('fourchanx-settings')) return Promise.reject(new Error(`close ${fam.name}'s settings window first`));
const run = async () => {
const root = document.documentElement;
root.classList.add('mbgx-xbusy');
let dialog = null;
try {
document.dispatchEvent(new CustomEvent('OpenSettings', { detail: 'Filter' }));
dialog = await waitFor(() => document.getElementById('fourchanx-settings'), 2000);
if (!dialog) throw new Error(`${fam.name} did not open its settings window`);
const credits = [...dialog.querySelectorAll('.credits a[target="_blank"]')];
// their name and version, as text, cut short (shown in messages only)
if (credits.length >= 2) xIdentity = { name: credits[0].textContent.trim().slice(0, 40), version: credits[1].textContent.trim().slice(0, 20) };
const select = await waitFor(() => dialog.querySelector('select[name="filter"]'), 2000);
if (!select || ![...select.options].some(o => o.value === 'MD5')) throw new Error(`${xFamily().name}: its Filter section has no Image MD5 list`);
if (select.value !== 'MD5') {
select.value = 'MD5';
select.dispatchEvent(new Event('change', { bubbles: true }));
}
const ta = await waitFor(() => dialog.querySelector('textarea[name="MD5"]'), 3000);
if (!ta) throw new Error(`${xFamily().name}: its Image MD5 list did not appear`);
const on = dialog.querySelector('input[name="Filter"]');
return await work(ta, { ...xFamily(), filterOff: !!on && !on.checked });
} finally {
const close = dialog && dialog.querySelector('.credits .close, a.close');
if (close && close.isConnected) close.click();
root.classList.remove('mbgx-xbusy');
}
};
// One operation at a time.
const p = (xBusy || Promise.resolve()).catch(() => {}).then(run);
xBusy = p.finally(() => { if (xBusy === p) xBusy = null; });
return p;
}
const waitFor = (fn, ms) => new Promise(resolve => {
const t0 = performance.now();
const tick = () => { const v = fn(); if (v || performance.now() - t0 > ms) resolve(v || null); else setTimeout(tick, 40); };
tick();
});
/** MD5s already in their list, active or commented out (a disabled rule is the user's decision). */
const xListed = text => new Set(String(text).split('\n').map(xLineMd5).filter(Boolean));
/**
* Append rules for files whose MD5 isn't in their list yet. items: [{md5, boards}].
* Resolves { added, already, name, version, filterOff }.
*/
function xAddMD5(items, stub) {
return withXFilter((ta, info) => {
const have = xListed(ta.value), lines = [], seen = new Set();
for (const { md5, boards } of items) {
const k = normMd5(md5);
if (!k || have.has(k) || seen.has(k)) continue;
seen.add(k);
lines.push(xRuleLine(k, boards, stub));
}
if (lines.length) {
ta.value = ta.value.replace(/\s+$/, '') + (ta.value.trim() ? '\n' : '') + lines.join('\n');
ta.dispatchEvent(new Event('change', { bubbles: true }));
}
for (const k of xListed(ta.value)) xKnown.add(k);
log('x-md5-added', { fork: info.name, version: info.version, added: lines.length, already: items.length - lines.length });
return { added: lines.length, already: items.length - lines.length, name: info.name, version: info.version, filterOff: info.filterOff };
});
}
const xKnown = new Set(); // MD5s seen in their list this page (saves reopening it for the same file)
/** Their whole MD5 list as text (for an explicit import). */
const xReadMD5 = () => withXFilter((ta, info) => ({ text: ta.value, name: info.name, version: info.version }));
/**
* Fallback when the settings window can't be used: XT, XZ and neXT's own Shift+click on the
* thumbnail ("MD5 Quick Filter in Threads"). Note it also hides the post with
* their own (stored) post hiding. Returns true if they handled the click.
*/
function xShiftClick(st) {
const link = st.post && st.post.querySelector(':scope > .file a.fileThumb');
if (!link) return false;
let unhandled = false;
const fallthrough = e => { unhandled = true; e.preventDefault(); }; // only reached if they didn't stop it
addEventListener('click', fallthrough);
link.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, shiftKey: true, view: window }));
removeEventListener('click', fallthrough);
log('x-md5-shift-click', { post: st.key, handled: !unhandled });
return !unhandled;
}
/** Add one post's file to their filter, by the settings window, else by Shift+click. Resolves a message or null. */
async function xAddPost(st, boards, cat) {
const fam = xFamily();
try {
const r = await xAddMD5([{ md5: st.md5, boards }], catSetting(cat, 'xNoStub') ? false : undefined);
return r.filterOff ? `${r.name} has filtering turned off (its Filter settings), so its MD5 list is not used until you turn it on.` : null;
} catch (e) {
log('x-md5-failed', { post: st.key, error: String(e && e.message || e) });
if (fam && fam.shiftClick && xShiftClick(st)) return null;
return `The MD5 was not added to ${fam ? fam.name : '4chan X'}'s filter: ${e && e.message || e}.`;
}
}
/**
* Opt-in: MD5s of files matched by picture (not by MD5) go to their list too.
* Collected and added at most once a minute while the page is visible, so their settings window
* isn't opened per post. Posts they already hide themselves are skipped. Off by default.
*/
const xAutoQueue = new Map();
let xAutoTimer = 0, xAutoLast = 0, xAutoError = null;
function xAutoAdd(st) {
const m = st.result && st.result.match;
if (!m || !catSetting(m.seed.cat, 'xAuto') || m.how === 'md5' || !st.md5 || xKnown.has(st.md5) || xAutoQueue.has(st.md5) || xAutoError || !xFamily()) return;
if (nativeHidden(st.target)) return;
xAutoQueue.set(st.md5, { boards: effective(m.seed).boards, noStub: catSetting(m.seed.cat, 'xNoStub') });
if (!xAutoTimer) xAutoTimer = setTimeout(xAutoFlush, Math.max(4000, 60000 - (Date.now() - xAutoLast)));
}
function xAutoFlush() {
xAutoTimer = 0;
if (!xAutoQueue.size) return;
if (document.visibilityState !== 'visible' || document.getElementById('fourchanx-settings')) { xAutoTimer = setTimeout(xAutoFlush, 10000); return; }
const queued = [...xAutoQueue];
xAutoQueue.clear();
xAutoLast = Date.now();
// one batch per stub choice (categories can differ), one after the other
let chain = Promise.resolve();
for (const noStub of [false, true]) {
const items = queued.filter(([, q]) => q.noStub === noStub).map(([md5, q]) => ({ md5, boards: q.boards }));
if (!items.length) continue;
chain = chain.then(() => xAddMD5(items, noStub ? false : undefined)).then(r => { stats.xAutoAdded += r.added; },
e => { xAutoError = String(e && e.message || e); log('x-auto-failed', { error: xAutoError, md5s: items.length }); });
}
}
/** The add-to-filter window behind every "⊘ blacklist" button. */
async function blacklist(st) {
if (st.busy) return;
st.busy = true;
renderButton(st);
try {
// The thumbnail is needed to match by picture; without it (deleted, failed) the exact file still can be.
let fp = null, hashError = null;
try { fp = await ensureHash(st); } catch (e) { hashError = e; }
if (!fp && !st.md5) throw hashError;
seeds = loadSeeds();
cats = loadCats();
const dup = fp && seeds.find(s => s.algo === fp.algo && s.a === fp.a && s.d === fp.d);
if (dup) { notify(`This image is already reference ${dup.id}${dup.cat && catById(dup.cat) ? ` (${catById(dup.cat).name})` : ''}.`); return; }
const md5Ref = st.md5 && seeds.find(s => !isPicture(s) && normMd5(s.md5) === st.md5);
const what = /^(webm|mp4)$/.test(st.ext || '') ? 'video' : isAnim(st) ? 'GIF' : 'image';
const hidden = (cfg.stubs ? 'hidden behind a stub' : 'hidden completely (stubs are off)') +
(cats.some(c => c.settings && typeof c.settings.stubs === 'boolean') ? ', unless the category you choose has its own stub setting' : '');
const catSel = categorySelect('mbgx-add-cat', catById(cfg.lastCat) ? cfg.lastCat : '');
const newName = el('input', { type: 'text', className: 'mbgx-add-newcat', placeholder: 'Name of the new category', maxLength: 60 });
const tags = el('input', { type: 'text', className: 'mbgx-add-tags', placeholder: 'optional, separated by commas' });
const boards = el('input', { type: 'text', className: 'mbgx-add-boards' });
const note = el('p', { className: 'mbgx-muted' });
const newRow = el('label', {}, 'New category ', newName);
// What to match: the picture (with edited copies, and this exact file), or only this exact file.
const kind = el('select', { className: 'mbgx-add-kind' },
fp && el('option', { value: 'picture', selected: !fp.low }, `This ${what} and edited copies of it`),
st.md5 && el('option', { value: 'md5', selected: !fp || fp.low }, 'Only this exact file'));
// Videos and GIFs: their thumbnail is their first frame, often black or one flat colour.
const weak = fp && isAnim(st) ? thumbnailWeakness(fp) : null;
const why = !fp ? 'The thumbnail could not be loaded, so only this exact file can be blacklisted.'
: weak === 'blank' ? `This ${what}'s thumbnail (its first frame) is almost blank, like the opening of many other ${what}s, so blacklisting the exact file is safer.`
: fp.low ? 'This thumbnail is almost blank and looks like many unrelated ones, so blacklisting the exact file is safer.' : '';
const warn = weak === 'weak' ? el('p', { className: 'mbgx-add-warn', role: 'note' }, `Warning: this ${what}'s thumbnail (its first frame) is ` +
`mostly one flat colour. Other ${what}s that open the same way could be filtered too. If that happens, mark them as false positives, ` +
'or blacklist only this exact file instead.')
: fp && fp.low && el('p', { className: 'mbgx-add-warn', role: 'note' }, `Warning: this ${isAnim(st) ? `${what}'s thumbnail (its first frame)` : 'thumbnail'} ` +
'is almost blank. Matched by picture, it also hides other posts whose thumbnails are almost blank in the same way (for example ' +
'videos that open on black). Mark any you want to keep as false positives, or blacklist only this exact file instead.');
// With a 4chan X-family script running, the MD5 can go to its filter as well.
const fam = st.md5 && xFamily();
const xBox = fam && el('input', { type: 'checkbox', className: 'mbgx-add-x', checked: cfg.md5Dest === 'both', onchange: () => { xTouched = true; } });
const keepPost = el('input', { type: 'checkbox', className: 'mbgx-add-post', checked: cfg.keepPost !== false });
let xTouched = false;
const sync = () => {
const c = catById(catSel.value);
if (xBox && !xTouched) xBox.checked = catSetting(c, 'xAdd'); // its category's choice, else the last one made here
newRow.hidden = catSel.value !== 'new';
if (warn) warn.hidden = kind.value === 'md5'; // only matters when matching by picture
boards.placeholder = c && c.boards ? `as its category: ${boardsText(c.boards)}` : 'all boards';
note.textContent = kind.value === 'md5' ? `Only identical copies of this file are filtered${c && c.boards ? ` (${scopeText(c.boards)})` : ''}.`
: `Uses ${c && c.rules ? `${c.name}'s own` : 'the general'} matching rules.`;
};
const form = el('div', { className: 'mbgx-form' },
el('label', {}, 'Match ', kind), why && el('p', { className: 'mbgx-muted mbgx-add-why' }, why),
warn,
md5Ref && el('p', { className: 'mbgx-muted' }, `This exact file is already blacklisted (${md5Ref.id}); "This ${what}…" makes it match edited copies too.`),
note,
el('label', {}, 'Category ', catSel), newRow,
el('label', {}, 'Tags ', tags), el('label', {}, 'Applies to ', boards),
el('label', {}, keepPost, ' Keep this post\'s number (for a link to its archived copy)'),
xBox && el('label', {}, xBox, ` Also add to ${fam.name}'s MD5 filter`));
for (const x of [catSel, kind]) x.addEventListener('change', sync);
sync();
if (!await ask(`Blacklist the ${what} in post No.${st.no}?\n\nMatching posts will be ${hidden}.`, 'Blacklist', 'Cancel', form)) return;
const own = parseBoards(boards.value);
if (!own) return;
if (cfg.keepPost !== keepPost.checked) { cfg = { ...cfg, keepPost: keepPost.checked }; write('settings', cfg); } // remembered
const where = xBox && xBox.checked ? 'both' : 'native';
const chosen = catById(catSel.value), ownX = chosen && chosen.settings && typeof chosen.settings.xAdd === 'boolean';
if (xBox && !ownX && cfg.md5Dest !== where) { cfg = { ...cfg, md5Dest: where }; write('settings', cfg); } // remembered unless its category decides
const existing = md5Ref && loadSeeds().find(s => s.id === md5Ref.id);
if (existing && kind.value === 'md5') { notify(`This exact file is already reference ${existing.id}.`); return; }
let message = null;
if (where === 'both') {
const c = catById(catSel.value);
message = await xAddPost(st, own.length ? own : c && c.boards ? c.boards : null, c);
}
let list = loadCats(), catId = catSel.value, created = null;
if (catId === 'new') {
[created, list] = categoryNamed(newName.value, list);
catId = created.id;
}
seeds = loadSeeds(); // the list may have changed while the dialog was open
const part = kind.value === 'picture' && st.px ? pictureOf(st.px) : null;
const pieces = kind.value === 'picture' && st.px ? piecesOf(st.px, fp) : [];
const picture = { ...(kind.value === 'picture' ? { algo: fp.algo, a: fp.a, d: fp.d, detail: st.px ? detailOf(integral(st.px)) : undefined,
...(fp.low ? { blank: true } : {}) } : {}),
...(part ? { part } : {}), ...(pieces.length ? { pieces: packPieces(pieces) } : {}) };
let seed;
if (existing) {
// An MD5-only reference of this file now matches by picture too (your choice here).
seeds = seeds.map(s => (s.id === existing.id ? (seed = cleanSeed({ ...s, ...picture, cat: catId || s.cat,
tags: [...(s.tags || []), ...parseTags(tags.value)], boards: own.length ? own : s.boards })) : s));
} else {
seed = cleanSeed({ id: nextSeedId(), ...picture, added: today(), board: st.board, thread: keepPost.checked ? st.thread : undefined,
post: keepPost.checked ? st.no : undefined, thumb: st.url, md5: st.md5, cat: catId || undefined, tags: parseTags(tags.value), boards: own });
seeds.push(seed);
}
if (created) write('categories', list);
write('seeds', seeds);
allow = loadAllow().filter(x => !(fp && x.a === fp.a && x.d === fp.d) && !(st.md5 && normMd5(x.md5) === st.md5));
write('allow', allow);
if (cfg.lastCat !== catId) { cfg = { ...cfg, lastCat: catId }; write('settings', cfg); }
log('seed-added', { post: st.key, seed: seed.id, kind: kind.value, cat: seed.cat, a: seed.a, d: seed.d, md5: !!seed.md5, url: st.url, upgraded: !!existing });
if (message) notify(message);
st.revealed = false;
rematchAll();
} catch (e) {
log('blacklist-error', { post: st.key, url: st.url, error: String(e && e.message || e) });
notify(`Could not hash this thumbnail:\n\n${e && e.message || e}`);
} finally {
st.busy = false;
renderButton(st);
flush();
}
}
/** Resolves true if the reference was removed. */
async function removeSeed(id) {
const seed = loadSeeds().find(s => s.id === id);
if (!seed || !await ask(`Remove reference ${id} from the blacklist?\n\nPosts hidden only because of it will reappear. This cannot be undone (export the blacklist first if unsure).`, 'Remove reference')) return false;
return !!guard('command', () => {
editSeeds(list => list.filter(s => s.id !== id), { removed: [id] });
return true;
});
}
/** Limit one reference to some boards, or with empty text to its category's / all boards. */
function setSeedBoards(id, text) {
const list = parseBoards(text);
if (!list) return false;
editSeeds(all => all.map(s => {
if (s.id !== id) return s;
const n = { ...s };
if (list.length) n.boards = list; else delete n.boards;
return n;
}), { boards: list, ids: [id] });
return true;
}
// ---------------------------------------------------------------- completing exact-file references
// A reference with only an MD5 catches that exact file and nothing else. Three ways give it the
// picture (hashes, detail, main picture and, if wanted, pieces), so reposts and edited copies are
// caught too: when it hides a post (that post's 4chan thumbnail, completeFromPost), and, when you
// ask, from a 4chan archive (lookupInArchive), offered after an import and from References.
/**
* Give reference `id` what matching by picture needs, from a thumbnail of exactly its file. An
* exact-file reference gets everything unless the thumbnail is nearly blank or mostly one colour
* (it would catch unrelated posts) or another reference already has this picture. A picture
* reference takes what it lacks only from its own picture (the same hashes). Returns 'added',
* 'completed', 'blank', 'same', 'other' or 'none'; saved at once.
*/
function completeReference(id, fp, px, { pieces = true, board = null } = {}) {
const list = loadSeeds(), s = list.find(x => x.id === id);
if (!s || !fp || !px) return 'none';
const add = {}, pic = isPicture(s);
if (pic && (s.a !== fp.a || s.d !== fp.d)) return 'other';
if (!pic) {
if (thumbnailWeakness(fp)) return 'blank';
if (list.some(x => x.id !== id && isPicture(x) && x.a === fp.a && x.d === fp.d)) return 'same';
Object.assign(add, { algo: fp.algo, a: fp.a, d: fp.d, w: fp.w, h: fp.h });
if (board && !s.board) add.board = board;
}
if (!s.detail) add.detail = detailOf(integral(px));
const part = !s.part && pictureOf(px);
if (part) add.part = part;
const pcs = pieces && !s.pieces && piecesOf(px, fp);
if (pcs && pcs.length) add.pieces = packPieces(pcs);
if (!Object.keys(add).length) return 'none';
const seed = cleanSeed({ ...s, ...add });
write('seeds', seeds = list.map(x => (x.id === id ? seed : x)));
return pic ? 'completed' : 'added';
}
/** An exact-file reference hid this post: its thumbnail gives the reference its picture (once per page). */
const completeTried = new Set();
let completeRematch = null;
function completeFromPost(st, seed) {
completeTried.add(seed.id);
const how = completeReference(seed.id, st.fp, st.px, { pieces: cfg.pieces !== false, board: st.board });
log('reference-completed', { post: st.key, seed: seed.id, how });
if (how !== 'added') return;
// the database is rebuilt once the current matching is done (several can complete in one go)
clearTimeout(completeRematch);
completeRematch = setTimeout(() => guard('rematch', () => { prepareData(); rematchAll(); }), 300);
}
/** References that a lookup can help: an MD5, and no picture yet (or no pieces, when pieces are wanted). */
const lookupNeeds = (s, pieces) => !!normMd5(s.md5) && (!isPicture(s) || (pieces && !s.pieces) || !s.detail);
/**
* Look references up in one archive, on one board, one file every 3 s: the archive's image search
* by MD5, then the thumbnail of a post whose file has exactly that MD5, hashed here as a 4chan
* thumbnail is. Only that archive's search address and its own image hosts are requested
* (archiveSearchUrl(), archiveThumbOk()). Runs until done or stopped; what was found is kept.
*/
let lookupJob = null;
const LOOKUP_GAP = 3000;
async function lookupInArchive(ids, board, hostIndex, pieces) {
const table = archives, host = table.hosts[hostIndex];
const job = lookupJob = { host, board, total: ids.length, done: 0, added: 0, completed: 0, notFound: 0, blank: 0, same: 0, failed: 0, stop: false, error: null };
const box = lookupProgress(job);
const sleep = ms => new Promise(res => setTimeout(res, ms));
const searchOk = u => typeof u === 'string' && u === archiveSearchUrl(table, hostIndex, board, (/image=([\w-]+)$/.exec(u) || [])[1]);
log('lookup-start', { host, board, references: ids.length, pieces });
let last = 0;
for (const id of ids) {
if (job.stop) break;
const s = loadSeeds().find(x => x.id === id);
if (!s || !lookupNeeds(s, pieces)) { job.done++; box.update(); continue; }
let found = null;
for (let attempt = 0; !job.stop; attempt++) {
await sleep(Math.max(0, last + LOOKUP_GAP - Date.now()));
last = Date.now();
try {
const answer = await withTimeout(gmRequestBlob(archiveSearchUrl(table, hostIndex, board, s.md5), searchOk), 30000, 'archive search');
if (answer.size > 2 * 1024 * 1024) throw new Error(`an answer of ${Math.round(answer.size / 1024)} KB, not a search result`); // a page of results is a few KB
let json;
try { json = JSON.parse(await answer.text()); } catch (e) { throw new Error('a page instead of a search result (it may be checking for bots)'); }
found = archiveFind(json, s.md5, host);
break;
} catch (e) {
const code = +((/HTTP (\d+)/.exec(e && e.message) || [])[1] || 0);
if (code === 404) break; // some archives answer "nothing found" this way
if ((code === 429 || code === 422 || code >= 500) && attempt < 2) { box.update(`${host} asked to slow down: waiting a minute…`); await sleep(60000); continue; }
job.error = `${host} answered: ${e && e.message || e}`;
job.stop = true;
}
}
if (job.stop) break;
if (!found) { job.notFound++; job.done++; box.update(); continue; }
try {
const blob = await withTimeout(gmRequestBlob(found.thumb, u => archiveThumbOk(u, host)), 30000, 'archive thumbnail');
const r = await hashBlob(blob);
const how = completeReference(id, r.fp, r.small, { pieces, board });
if (how === 'added') job.added++;
else if (how === 'completed') job.completed++;
else if (how === 'blank') job.blank++;
else if (how === 'same') job.same++;
else if (how === 'other') job.failed++;
} catch (e) {
job.failed++;
}
job.done++;
box.update();
}
lookupJob = null;
log('lookup-done', { host, board, done: job.done, added: job.added, completed: job.completed, notFound: job.notFound, blank: job.blank, same: job.same, failed: job.failed, error: job.error });
prepareData();
rematchAll();
box.remove();
notify(lookupSummary(job));
}
const lookupSummary = j => `Archive lookup on ${j.host} (/${j.board}/)${j.stop && !j.error ? ', stopped' : ''}: ${j.done} of ${j.total} looked up.` +
`\n\n${j.added} now match by picture too${j.completed ? `, ${j.completed} gained what they lacked (pieces, detail)` : ''}; ` +
`${j.notFound} not found on that board.` +
(j.blank ? ` ${j.blank} kept as exact file only: their thumbnail is nearly blank or mostly one colour, so matching it would hide unrelated posts.` : '') +
(j.same ? ` ${j.same} skipped: another reference already has that picture.` : '') +
(j.failed ? ` ${j.failed} could not be used (thumbnail missing or different).` : '') +
(j.error ? `\n\nStopped: ${j.error}. Try again later, or another archive.` : '') +
(j.done < j.total ? '\n\nLooking up again carries on with the rest.' : '');
/** A small box with the lookup's progress and a Stop button, until it ends. */
function lookupProgress(job) {
const text = el('span', {});
const box = el('div', { className: 'mbgx-panel mbgx-lookup', role: 'status' }, text, ' ',
el('button', { type: 'button', className: 'mbgx-lookup-stop', onclick: () => { job.stop = true; text.textContent = 'Stopping…'; } }, 'Stop'));
guard('control', applyTheme);
document.body.append(box);
box.update = msg => {
if (job.stop) return;
text.textContent = msg || `Looking up in ${job.host} (/${job.board}/): ${job.done} of ${job.total} · ${job.added + job.completed} completed · ${job.notFound} not found`;
};
box.update();
return box;
}
/** The lookup window: which board and archive, pieces or not, and what it means. `ids`: the references to consider. */
async function lookupDialog(ids, intro = '') {
if (lookupJob) { notify('An archive lookup is already running on this page.'); return; }
const list = loadSeeds().filter(s => ids.includes(s.id));
if (!list.some(s => lookupNeeds(s, true))) {
notify(list.length ? 'Nothing to look up: these references already match by picture, with pieces, or have no MD5 to search for.' : 'No references to look up.');
return;
}
// the board most of them come from (their own, else where they apply), else this page's
const tally = new Map();
for (const s of list) {
const c = catById(s.cat), b = s.board || (s.boards || [])[0] || (c && (c.boards || [])[0]);
if (b) tally.set(b, (tally.get(b) || 0) + 1);
}
const board0 = [...tally].sort((x, y) => y[1] - x[1]).map(([b]) => b)[0] || BOARD || '';
const boardBox = el('input', { type: 'text', className: 'mbgx-lookup-board', value: board0, maxLength: 12, size: 6 });
const archiveSel = el('select', { className: 'mbgx-lookup-archive' });
const pieces = el('input', { type: 'checkbox', className: 'mbgx-lookup-pieces', checked: cfg.pieces !== false });
const count = el('p', { className: 'mbgx-muted mbgx-lookup-count' });
const fill = () => {
const b = boardBox.value.trim().replace(/\//g, '').toLowerCase();
const cands = (Object.prototype.hasOwnProperty.call(archives.boards, b) ? archives.boards[b] : []).filter(([i]) => archives.ff[i]);
const pick = pickArchives(archives, b, cfg.archivePick[b]);
archiveSel.replaceChildren(...(cands.length ? cands.map(([i]) => el('option', { value: i, selected: pick && pick.search === i }, archives.hosts[i]))
: [el('option', { value: '' }, 'no archive with image search for this board')]));
const n = list.filter(s => lookupNeeds(s, pieces.checked)).length;
count.textContent = `${refCount(n)} to look up: about ${Math.max(1, Math.round(n * LOOKUP_GAP / 60000))} minute${n * LOOKUP_GAP < 90000 ? '' : 's'}` +
` (one file every ${LOOKUP_GAP / 1000} s). References without an MD5 can't be looked up.`;
};
boardBox.addEventListener('input', fill);
pieces.addEventListener('change', fill);
fill();
if (!await ask(`${intro}Look up references in a 4chan archive?\n\n` +
'Each file is searched for by its MD5 on the archive you choose. When a post with exactly that file is found, its thumbnail is downloaded and ' +
'added to the reference here, so it also catches reposts and edited copies.\n\n' +
'This connects to that archive: it sees your IP address and which files you look up. It can take a while; you can stop at any time, ' +
'and what was found is kept.',
'Look up', 'Cancel', el('div', { className: 'mbgx-form' },
el('label', {}, 'Board ', boardBox), el('label', {}, 'Archive ', archiveSel),
el('label', {}, pieces, ' Also add pieces (catch heavily cropped copies)'), count))) return;
const board = boardBox.value.trim().replace(/\//g, '').toLowerCase(), hostIndex = archiveSel.value === '' ? -1 : +archiveSel.value;
if (hostIndex < 0 || !archiveSearchUrl(archives, hostIndex, board, 'AAAAAAAAAAAAAAAAAAAAAA==')) { notify('Choose a board that an archive with image search keeps.'); return; }
const todo = list.filter(s => lookupNeeds(s, pieces.checked)).map(s => s.id);
lookupInArchive(todo, board, hostIndex, pieces.checked).catch(e => { lookupJob = null; notify(`Archive lookup failed: ${e && e.message || e}`); });
}
/** After an import that added exact-file references: the import's message, and the offer to look them up. */
async function offerLookup(message, ids) {
if (!ids || !ids.length) return message;
if (await ask(`${message}\n\n${refCount(ids.length)} added here match${ids.length === 1 ? 'es' : ''} exact files only. ` +
'Look them up in a 4chan archive, so reposts and edited copies are caught too? This connects to the archive and can take a while.',
'Look up…', 'Not now')) await lookupDialog(ids);
return null;
}
// ---------------------------------------------------------------- sharing: export and import
/** JSON that stays readable but compact: one entry per line instead of one field per line. */
function exportText(data) {
return '{\n' + Object.entries(data).map(([k, v]) => ` ${JSON.stringify(k)}: ` + (Array.isArray(v) && v.length && typeof v[0] === 'object'
? '[\n' + v.map(x => ' ' + JSON.stringify(x)).join(',\n') + '\n ]' : JSON.stringify(v))).join(',\n') + '\n}\n';
}
// What each export window offers. `needs`: a toggle that only works with another one on.
const EXPORTS = {
references: {
title: 'Export blacklist', file: 'better-filter-blacklist.json', what: 'reference', items: 'references',
always: 'Always included: the hash method and the two image hashes (for exact-file references, the MD5), which is all the filter needs.',
toggles: [
['source', 'Image MD5 and source board', "Lets whoever imports it match the exact files and search the board's archive for them. Shows which board each image came from."],
['post', 'Exact post', 'Post and thread number: imported references link to the post and its archive copy. Shows exactly which posts you blacklisted.', 'source'],
['boards', 'Board restrictions', "Where each reference applies, including its category's boards. Off: everything exported applies to all boards."],
['dates', 'Date added', 'When you blacklisted each image. Off: the importer sees the date of the import.'],
['cats', 'Categories and tags', 'Category names, their settings and your tags. Off: the names you chose stay private.'],
['pieces', 'Pieces', 'Lets the importer catch heavily cropped copies too, about 1 KB per reference. Off: a smaller file; references get pieces when their exact image is seen again.'],
],
make: (f, list) => blacklistFile(list, f, cats),
},
falsePositives: {
title: 'Export false positives', file: 'better-filter-false-positives.json', what: 'false positive', items: 'falsePositives',
always: 'Always included: the two image hashes, which is all the whitelist needs.',
toggles: [
['post', 'Post where it happened', 'Board, thread and post number of each false positive, and the MD5 of its file.'],
['match', 'What it matched', 'The hashes of the reference it was mistaken for, and the two distances. Linked to the importer\'s copy of that reference if they have it.'],
['dates', 'Date added', 'When you marked each one. Off: the importer sees the date of the import.'],
],
make: (f, list) => falsePositivesFile(list, loadSeeds(), f),
},
};
/**
* Export window: choose which entries (scopes: [{label, list}], first is preselected) and what goes
* into the file, with a preview; choices are remembered. Everything on = entries arrive as if the
* importer had added them; everything off = hashes only.
*/
function exportDialog(kind, scopes) {
if (document.getElementById('mbgx-export')) return;
const E = EXPORTS[kind], f = { ...DEFAULTS.exportFields[kind], ...(cfg.exportFields[kind] || {}) };
scopes = scopes || [{ label: 'All', list: kind === 'references' ? loadSeeds() : loadAllow() }];
let scope = scopes[0];
const opts = () => Object.fromEntries(E.toggles.map(([k, , , needs]) => [k, !!f[k] && (!needs || !!f[needs])]));
const toggle = ([key, label, hint]) => el('div', { className: 'mbgx-exp-row' },
el('label', {}, el('input', { type: 'checkbox', className: `mbgx-exp-${key}`, checked: !!f[key], onchange: e => { f[key] = e.target.checked; update(); } }), ' ', label),
el('div', { className: 'mbgx-muted' }, hint));
const preview = el('pre', { className: 'mbgx-live' });
const close = () => {
removeEventListener('keydown', onKey, true);
backdrop.remove();
dlg.remove();
};
// Window-level capture runs before the panel's own Escape handler, so Escape closes only this window.
const onKey = e => {
if (e.key !== 'Escape') return;
e.stopPropagation();
close();
};
const backdrop = el('div', { className: 'mbgx-backdrop mbgx-export-backdrop', onclick: close });
guard('control', applyTheme);
const dlg = el('div', { id: 'mbgx-export', className: 'mbgx-panel mbgx-export', dataset: { kind } },
el('h3', {}, E.title),
scopes.length > 1 && el('p', {}, el('label', {}, 'Export ', el('select', { className: 'mbgx-exp-scope', onchange: e => { scope = scopes[+e.target.value]; update(); } },
scopes.map((sc, i) => el('option', { value: i }, `${sc.label} (${sc.list.length})`))))),
el('p', { className: 'mbgx-muted' }, E.always),
E.toggles.map(toggle),
preview,
el('p', {},
el('button', { type: 'button', className: 'mbgx-exp-go', onclick: () => {
cfg = { ...cfg, exportFields: { ...cfg.exportFields, [kind]: { ...f } } };
write('settings', cfg);
download(E.file, exportText(E.make(opts(), scope.list)));
log('exported', { kind, count: scope.list.length, fields: opts() });
close();
} }, 'Export'),
el('button', { type: 'button', onclick: close }, 'Cancel')));
const update = () => {
for (const [key, , , needs] of E.toggles) {
if (!needs) continue;
const box = dlg.querySelector(`.mbgx-exp-${key}`);
box.disabled = !f[needs];
box.closest('.mbgx-exp-row').classList.toggle('mbgx-disabled', !f[needs]);
}
const file = E.make(opts(), scope.list), items = file[E.items];
const size = new Blob([exportText(file)]).size;
preview.textContent = `${items.length} ${E.what}${items.length === 1 ? '' : 's'}, ${size < 10240 ? size + ' bytes' : Math.round(size / 1024) + ' KB'}. ` +
(items.length ? `First entry:\n${JSON.stringify(items[0], null, 1)}` : 'Nothing to export.');
};
update();
document.body.append(backdrop, dlg);
addEventListener('keydown', onKey, true);
}
const sameBoards = (x, y) => !!x && !!y && x.length === y.length && x.every(b => y.includes(b));
/**
* Import a blacklist file (this script's export, the list from "Edit references as JSON", or a full
* diagnostics export). target: {mode: 'file'} = the file's categories, merged by name with existing ones;
* {mode: 'cat', id} = everything into one category ('' = Uncategorised). Rules, so an import never
* makes an existing filter apply more widely:
* - a fingerprint you already have is kept as it is (its tags gain the file's tags);
* - a file category with the same name as one of yours uses yours, settings unchanged;
* - a reference keeps the boards it applied to in the file; it only inherits its category's
* boards when they are the same.
*/
function importBlacklist(text, target = { mode: 'file' }) {
let data = JSON.parse(text), fileCats = [];
if (data && !Array.isArray(data)) {
fileCats = fileCategories(data);
data = data.references || data.seeds;
}
if (!Array.isArray(data)) throw new Error('No reference list found in this file.');
const current = loadSeeds(), byHash = new Map(current.filter(isPicture).map(s => [`${s.algo} ${s.a} ${s.d}`, s]));
const byMd5 = new Map(current.filter(s => normMd5(s.md5)).map(s => [normMd5(s.md5), s]));
let list = loadCats();
const listBefore = list.length, local = new Map();
const result = { added: 0, duplicate: 0, invalid: 0, renamed: 0, tagged: 0, newCats: 0, keptSettings: 0, md5Only: 0, md5OnlyIds: [], ownSettings: [], completed: 0 };
// IDs in use, and the next free number: kept as we go (a check over the whole list per reference
// would take minutes for a file of hundreds of thousands)
const ids = new Set(current.map(s => s.id));
let nextNo = +nextSeedId(current).slice(1);
const localCat = i => {
if (!local.has(i)) {
const fc = fileCats[i] || {}, exists = list.find(c => c.name.toLowerCase() === String(fc.name || '').trim().slice(0, 60).toLowerCase());
let c;
[c, list] = categoryNamed(fc.name || 'Imported', list);
if (exists) {
const theirs = portableSettings(cleanCatSettings(fc)).settings || {}, ours = portableSettings(c.settings).settings || {};
if (!sameBoards(c.boards || [], fc.boards || []) || JSON.stringify(c.rules || null) !== JSON.stringify(cleanRules(fc.rules)) ||
JSON.stringify(theirs) !== JSON.stringify(ours)) result.keptSettings++;
} else {
list = list.map(x => (x.id === c.id ? (c = cleanCat({ ...c, enabled: fc.enabled !== false, boards: fc.boards, rules: fc.rules,
settings: portableSettings(cleanCatSettings(fc)).settings })) : x));
if (c.rules || c.settings) result.ownSettings.push(c.name);
}
local.set(i, c);
}
return local.get(i);
};
let fixed = target.mode === 'cat' && target.id ? list.find(c => c.id === target.id) : null;
if (target.mode === 'new') [fixed, list] = categoryNamed(target.name, list);
for (const x of data) {
// A picture reference (hashes) or an exact-file one (only an MD5).
const pic = isPicture(x), md5 = x && normMd5(x.md5);
if (!pic && !md5) { result.invalid++; continue; }
const have = pic ? byHash.get(`${x.algo} ${x.a} ${x.d}`) : byMd5.get(md5);
if (have) {
const tags = Array.isArray(x.tags) ? x.tags.filter(t => !(have.tags || []).includes(t)) : [];
if (tags.length) { Object.assign(have, cleanSeed({ ...have, tags: [...(have.tags || []), ...tags] })); result.tagged++; }
if (!have.detail && have.a && DETAIL_TEXT.test(x.detail || '')) have.detail = x.detail;
if (!have.part && have.a && x.part) { const c = cleanSeed({ ...have, part: x.part }); if (c.part) have.part = c.part; }
if (!have.pieces && have.a && x.pieces) { const pt = piecesText(x.pieces); if (pt) have.pieces = pt; }
// What the file knows about it and the list doesn't: its MD5, source board and post. Only
// filled in, never overwritten; a post only with its own board.
let filled = false;
if (md5 && !have.md5 && !byMd5.has(md5)) { have.md5 = md5; byMd5.set(md5, have); filled = true; }
const board = /^[a-z0-9]{1,12}$/.test(x.board) ? x.board : null;
if (board && !have.board) { have.board = board; filled = true; }
if (board && have.board === board && !have.post && /^\d{1,12}$/.test(String(x.post || ''))) {
have.post = String(x.post);
if (!have.thread && /^\d{1,12}$/.test(String(x.thread || ''))) have.thread = String(x.thread);
filled = true;
}
if (filled) result.completed++;
result.duplicate++;
continue;
}
const seed = cleanSeed({ ...x, cat: undefined });
if (!seed.added) seed.added = today();
if (typeof x.id !== 'string' || !SEED_ID.test(x.id) || ids.has(seed.id)) {
if (typeof x.id === 'string') result.renamed++;
while (ids.has(`r${nextNo}`)) nextNo++;
seed.id = `r${nextNo++}`;
}
if (!pic) { result.md5Only++; result.md5OnlyIds.push(seed.id); }
ids.add(seed.id);
const c = fixed || (target.mode === 'file' && Number.isInteger(x.cat) && x.cat >= 0 && fileCats[x.cat] ? localCat(x.cat) : null);
if (c) {
seed.cat = c.id;
if (sameBoards(seed.boards, c.boards)) delete seed.boards;
}
current.push(seed);
if (pic) byHash.set(`${seed.algo} ${seed.a} ${seed.d}`, seed);
if (seed.md5 && !byMd5.has(seed.md5)) byMd5.set(seed.md5, seed);
result.added++;
}
result.newCats = list.length - listBefore;
// All or nothing: if the references can't be saved (storage full, say), the categories go back too.
const catsBefore = read('categories', []);
if (result.newCats) write('categories', list);
try { write('seeds', current); } catch (e) { if (result.newCats) write('categories', catsBefore); throw e; }
log('seeds-imported', result);
rematchAll();
flush();
return result;
}
/**
* A file's categories, as far as they are usable: objects with a name (text, cut to 60 characters),
* at most 500 (references pointing past those are imported Uncategorised).
*/
function fileCategories(data) {
return (Array.isArray(data && data.categories) ? data.categories : []).slice(0, 500)
.map(c => (c && typeof c === 'object' && typeof c.name === 'string' && c.name.trim() ? { ...c, name: c.name.trim().slice(0, 60) } : null));
}
/** What a reference's ID may be when it comes from outside (a file, the JSON editor): short plain text. */
const SEED_ID = /^[A-Za-z0-9_-]{1,40}$/;
/** A whitelist entry's identity: its picture hashes, else its MD5. */
const fpKey = x => (x.a ? `${x.a} ${x.d}` : `md5 ${normMd5(x.md5)}`);
/** False-positive file import: merges, skipping fingerprints already on the list. */
function importFalsePositives(items) {
const current = loadAllow(), refs = loadSeeds(), result = { added: 0, duplicate: 0, invalid: 0 };
const have = new Set(current.map(fpKey));
for (const x of items) {
const pic = !!x && HEX64.test(x.a) && HEX64.test(x.d), md5 = x && normMd5(x.md5);
if (!pic && !md5) { result.invalid++; continue; }
if (have.has(fpKey(x))) { result.duplicate++; continue; }
const e = pic ? { a: x.a, d: x.d, ...(md5 ? { md5 } : {}) } : { md5 };
if (/^[a-z0-9]{1,12}\.\d{1,12}$/.test(x.post || '')) {
e.post = x.post;
if (/^\d{1,12}$/.test(String(x.thread || ''))) e.thread = String(x.thread);
}
const m = x.matched && refs.find(r => (x.matched.a ? r.a === x.matched.a && r.d === x.matched.d : normMd5(r.md5) === normMd5(x.matched.md5)));
if (m) e.seed = m.id;
if ([x.da, x.dd].every(v => Number.isInteger(v) && v >= 0 && v <= 64)) { e.da = x.da; e.dd = x.dd; }
e.added = typeof x.added === 'string' && /^\d{4}-\d{2}-\d{2}/.test(x.added) ? x.added.slice(0, 10) : today();
current.push(e);
have.add(fpKey(e));
result.added++;
}
write('allow', current);
log('false-positives-imported', result);
rematchAll();
flush();
return result;
}
/** Import either kind of exported file, recognised by its contents; asks where references should go. */
async function importFile(text, intoCat) {
const data = JSON.parse(text);
if (data && Array.isArray(data.falsePositives)) {
const r = importFalsePositives(data.falsePositives);
return `False positives imported: ${r.added} added, ${r.duplicate} already present, ${r.invalid} invalid`;
}
const refs = Array.isArray(data) ? data : data && (data.references || data.seeds) || [];
const fileCats = fileCategories(data).filter(Boolean);
const catNames = fileCats.slice(0, 10).map(c => c.name).join(', ') + (fileCats.length > 10 ? ` and ${fileCats.length - 10} more` : '');
cats = loadCats();
const where = el('select', { className: 'mbgx-import-into' },
fileCats.length ? el('option', { value: 'file', selected: intoCat === undefined }, `Their own categories (${catNames})`) : '',
el('option', { value: '', selected: intoCat === '' || (intoCat === undefined && !fileCats.length) }, 'Uncategorised'),
cats.map(c => el('option', { value: c.id, selected: intoCat === c.id }, c.name)),
el('option', { value: 'new' }, '+ New category…'));
const newName = el('input', { type: 'text', className: 'mbgx-import-newcat', placeholder: 'Name of the new category', maxLength: 60 });
const newRow = el('label', { hidden: true }, 'New category ', newName);
where.addEventListener('change', () => { newRow.hidden = where.value !== 'new'; });
if (!await ask(`Import ${refs.length} reference${refs.length === 1 ? '' : 's'}?\n\nReferences you already have are left as they are. References keep the boards they apply to in the file.`,
'Import', 'Cancel', el('div', { className: 'mbgx-form' }, el('label', {}, 'Into ', where), newRow))) return null;
let target = { mode: 'file' };
if (where.value === 'new') target = { mode: 'new', name: newName.value }; // made by the import, saved only with it
else if (where.value !== 'file') target = { mode: 'cat', id: where.value };
const r = importBlacklist(text, target);
return offerLookup(`Import: ${r.added} added, ${r.duplicate} already present, ${r.invalid} invalid` +
(r.renamed ? `, ${r.renamed} renumbered to avoid ID clashes` : '') +
(r.completed ? `\n\n${r.completed} reference${r.completed === 1 ? '' : 's'} you already had gained ${r.completed === 1 ? 'its' : 'their'} source post or MD5 from the file.` : '') +
(r.newCats ? `\n\n${r.newCats} new categor${r.newCats === 1 ? 'y' : 'ies'} created.` : '') +
(r.keptSettings ? `\n\n${r.keptSettings} categor${r.keptSettings === 1 ? 'y' : 'ies'} already existed with different settings; yours were kept.` : '') +
(r.ownSettings.length ? `\n\nNew categor${r.ownSettings.length === 1 ? 'y' : 'ies'} with their own matching limits or settings from the file: ` +
`${r.ownSettings.slice(0, 10).join(', ')}${r.ownSettings.length > 10 ? ` and ${r.ownSettings.length - 10} more` : ''}. ` +
'Check them in References (choose the category) → Settings before relying on them.' : '') +
(r.tagged ? `\n\n${r.tagged} existing reference${r.tagged === 1 ? '' : 's'} gained tags from the file.` : '') +
(r.md5Only ? `\n\n${r.md5Only} of the added references match exact files only (MD5).` : ''), r.md5OnlyIds);
}
/**
* Import an X-family MD5 filter (pasted, or read from their settings window): each rule becomes an
* exact-file reference in `catId` ('' = Uncategorised), keeping its boards. Rules that would filter
* differently here are left out and counted (parseXFilter()). Files you already have are skipped.
*/
function importXFilter(text, catId, newName) {
const parsed = parseXFilter(text), current = loadSeeds();
// "+ New category": made here and saved only with the references it gets (all or nothing)
let newList = null, cat = catById(catId);
if (catId === 'new') { [cat, newList] = categoryNamed(newName, loadCats()); }
const have = new Set(current.map(x => normMd5(x.md5)).filter(Boolean));
const result = { added: 0, duplicate: 0, invalid: parsed.invalid, skipped: parsed.skipped, ids: [] };
let nextNo = +nextSeedId(current).slice(1); // counted on, not searched for per rule: a long filter stays quick
for (const r of parsed.rules) {
if (have.has(r.md5)) { result.duplicate++; continue; }
const seed = cleanSeed({ id: `r${nextNo++}`, md5: r.md5, added: today(), cat: cat ? cat.id : undefined, boards: r.boards || undefined });
if (cat && sameBoards(seed.boards, cat.boards)) delete seed.boards;
current.push(seed);
have.add(r.md5);
result.ids.push(seed.id);
result.added++;
}
if (!result.added) return result; // nothing to save, no empty category
const catsBefore = read('categories', []);
if (newList) write('categories', newList);
try { write('seeds', current); } catch (e) { if (newList) write('categories', catsBefore); throw e; }
log('x-filter-imported', { added: result.added, duplicate: result.duplicate, invalid: result.invalid, skipped: result.skipped });
rematchAll();
flush();
return result;
}
const importXText = r => `MD5 filter import: ${r.added} added, ${r.duplicate} already present, ${r.invalid} not an MD5 rule` +
(Object.keys(r.skipped).length ? `\n\nLeft out (they would filter differently here): ${Object.entries(r.skipped).map(([k, n]) => `${n} ${k}`).join(', ')}.` : '');
/** Ask where an X-family MD5 filter goes, then import it. `text` null: paste it here. */
async function importXDialog(text, source) {
cats = loadCats();
const box = text === null && el('textarea', { className: 'mbgx-xf-text', spellcheck: false, placeholder: '/xxxxxxxxxxxxxxxxxxxxxx==/\n/yyyyyyyyyyyyyyyyyyyyyy==/;boards:vg;' });
const into = categorySelect('mbgx-xf-cat', '', true);
const newName = el('input', { type: 'text', className: 'mbgx-xf-newcat', placeholder: 'Name of the new category', maxLength: 60 });
const newRow = el('label', { hidden: true }, 'New category ', newName);
into.addEventListener('change', () => { newRow.hidden = into.value !== 'new'; });
const n = text === null ? null : parseXFilter(text).rules.length;
if (!await ask(text === null ? "Import a 4chan X / XT / XZ / neXT MD5 filter\n\nPaste the rules from their Settings → Filter → Image MD5. Each becomes an exact-file reference here; rules with options Better Filter can't follow are left out and counted."
: `Import ${n} MD5 rule${n === 1 ? '' : 's'} from ${source}?\n\nEach becomes an exact-file reference here, keeping its boards. Their filter is not changed.`,
'Import', 'Cancel', el('div', { className: 'mbgx-form' }, box, el('label', {}, 'Into ', into), newRow))) return null;
const r = importXFilter(text === null ? box.value : text, into.value, newName.value);
return offerLookup(importXText(r), r.ids);
}
function chooseImportFile(done, intoCat) {
const input = el('input', { type: 'file', accept: '.json,application/json', style: 'display:none' });
input.addEventListener('change', () => {
const file = input.files[0];
input.remove();
if (!file) return;
// 64 MB: about 45,000 references with every field; anything larger isn't an export of this script
if (file.size > 64 * 1024 * 1024) { notify(`Import refused: ${Math.round(file.size / 1048576)} MB is far larger than any blacklist file.`); return; }
file.text().then(text => importFile(text, intoCat)).then(msg => {
if (msg) notify(msg);
done();
}).catch(e => notify(`Import failed: ${e && e.message || e}`));
});
document.body.append(input);
input.click();
}
// ---------------------------------------------------------------- post commands
function revealPost(st) {
const m = st.result && st.result.match;
st.revealed = true;
stats.revealed++;
unstub(st.target);
log('revealed', { post: st.key, seed: m && m.seed.id, da: m && m.da, dd: m && m.dd });
renderButton(st);
updateControl();
}
function rehidePost(st) {
const m = st.result && st.result.match;
st.revealed = false;
log('rehidden', { post: st.key, seed: m && m.seed.id });
applyMatch(st);
}
function whitelistPost(st, m) {
const all = loadAllow();
all.push({ ...(st.fp ? { a: st.fp.a, d: st.fp.d } : {}), ...(st.md5 ? { md5: st.md5 } : {}), post: st.key, thread: st.thread,
seed: m && m.seed.id, ...(m && m.da != null ? { da: m.da, dd: m.dd } : {}), added: today() });
write('allow', all);
stats.whitelisted++;
log('false-positive', { post: st.key, url: st.url, seed: m && m.seed.id, how: m && (m.how || 'hash'), da: m && m.da, dd: m && m.dd, a: st.fp && st.fp.a, d: st.fp && st.fp.d, md5: !!st.md5 });
rematchAll();
flush();
}
function onCommand(target) {
if (target.dataset.mbgxCmd === 'reveal-reply') {
revealedReplies.add(target.dataset.mbgxNo);
scheduleQuotes();
return;
}
const st = posts.get(target.dataset.mbgxKey);
if (!st) return;
const name = target.dataset.mbgxCmd || 'action';
const m = st.result && st.result.match;
if (name === 'reveal') {
const root = target.closest('.postContainer');
if (st.role === 'reply' && root && root !== st.root) { unstub(root); return; } // a clone (XT inline quote): reveal just that copy
revealPost(st);
} else if (name === 'whitelist') {
if (!st.fp && !st.md5) return;
// An exact-file match came before any hashing: hash it now (when picture references exist), so the
// same picture isn't filtered by picture straight after.
ask(`Mark post No.${st.no} as a false positive?\n\nThis exact ${m && m.how === 'md5' ? 'file and picture' : 'image fingerprint'} will never be filtered again. Reference ${m ? m.seed.id : '?'} stays active.`, 'Mark false positive')
.then(ok => ok && (st.fp || !activeSeeds().length ? Promise.resolve() : ensureHash(st).catch(() => null))
.then(() => guard('command', () => whitelistPost(st, m))));
} else if (name === 'unseed') {
if (m) removeSeed(m.seed.id);
} else if (m && st.revealed) {
rehidePost(st);
} else {
blacklist(st);
}
}
// One capture-phase listener handles every Better Filter control, including copies inside XT clones.
document.addEventListener('click', e => {
const t = e.target.closest && e.target.closest('.mbgx-action, .mbgx-cmd');
if (!t) return;
e.preventDefault();
e.stopPropagation();
guard('command', () => onCommand(t));
}, true);
document.addEventListener('keydown', e => {
if ((e.key === 'Enter' || e.key === ' ') && e.target.matches && e.target.matches('.mbgx-action, .mbgx-cmd')) {
e.preventDefault();
guard('command', () => onCommand(e.target));
}
}, true);
// ---------------------------------------------------------------- settings panel
function download(name, text) {
const url = URL.createObjectURL(new Blob([typeof text === 'string' ? text : JSON.stringify(text, null, 2)], { type: 'application/json' }));
const a = el('a', { href: url, download: name, style: 'display:none' });
document.body.append(a);
a.click();
a.remove();
setTimeout(() => URL.revokeObjectURL(url), 5000);
}
/**
* Memory the script holds for this page's pictures. Measured: the greyscale copies kept per post
* (exact byte counts). Estimated: the pixel buffers alive at the busiest moment (decodes in flight at
* once x the largest thumbnail's RGBA bytes; each is released as soon as it is hashed). The browser's own
* JavaScript heap is only measurable in Chromium (page.browser.jsHeapPeak).
*/
function memoryEstimate() {
let grayBytes = 0, withGray = 0;
for (const st of posts.values()) if (st.px) { grayBytes += st.px.g.byteLength; withGray++; }
return { measured: { grayRetainedBytes: grayBytes, postsWithGray: withGray, canvases: stats.canvases, imageBitmaps: stats.bitmaps, peakDecodes: stats.peakDecodes },
estimated: { peakPixelBufferBytes: stats.peakDecodes * stats.largestPixels * 4 } };
}
function diagnostics() {
flush();
const states = [...posts.values()];
const pick = st => ({ post: st.key, role: st.role, url: st.url, status: st.status, error: st.error, a: st.fp && st.fp.a, d: st.fp && st.fp.d,
best: st.result && st.result.best && { seed: st.result.best.seed.id, da: st.result.best.da, dd: st.result.best.dd },
matched: !!(st.result && st.result.match), revealed: st.revealed, stubbed: !!(st.target && st.target.classList.contains('mbgx-filtered')) });
return {
script: '4chan Better Filter', version: VERSION, exported: new Date().toISOString(),
environment: environment(), canvasSelfTest: session.canvas, settings: cfg, categories: cats, seeds, allow,
page: {
url: location.href, kind: PAGE, stats: statsOut(), queueLength: queue.length, activeRequests: active,
workers: { state: pool.state, count: pool.workers.length, reason: pool.reason, preparedMs: pool.indexMs, stopped: pool.retired, waiting: pool.waiting.length },
browser: browserStats, memory: memoryEstimate(),
matched: states.filter(s => s.result && s.result.match).map(pick),
nearMisses: states.filter(s => s.result && s.result.near).map(pick),
errors: states.filter(s => s.status === 'error').map(pick),
pending: states.filter(s => s.queued || s.status === 'hashing').map(pick).slice(0, 50),
},
// with the stored log off: this page's summary and entries only
sessions: cfg.keepLog ? read('sessions', []) : [{ ...session, end: new Date().toISOString(), stats: statsOut() }],
logs: cfg.keepLog ? read('logs', []).concat(pendingLogs) : pageLog.slice(),
};
}
/**
* Diagnostics without anything about what you looked at or blacklisted (the default export,
* fit to share): no page addresses, boards, post or thread numbers, image hashes or MD5s, references,
* false positives, category names or tags, and no times of day; the browser as name and version only.
* Kept: script and browser versions, settings, counts, timings and error messages (addresses and long
* numbers in them replaced).
*/
function anonymousDiagnostics() {
const d = diagnostics();
const kindOf = url => { try { const p = new URL(url).pathname; return /\/thread\//.test(p) ? 'thread' : /\/catalog/.test(p) ? 'catalog' : 'index'; } catch (e) { return 'unknown'; } };
const scrub = t => String(t || '').replace(/https?:\/\/\S+/g, '[address]').replace(/\b\d{5,}\b/g, '[number]');
const ua = /(Firefox|Edg|Chrome|Safari)\/(\d+)/.exec(d.environment.userAgent || '');
const { userAgent, ...env } = d.environment;
const { lastCat, archivePick, ...settings } = d.settings;
const byType = {};
for (const l of d.logs) byType[l.type] = (byType[l.type] || 0) + 1;
return {
script: d.script, version: d.version, anonymised: true, exported: d.exported.slice(0, 10),
environment: { ...env, browser: ua ? `${ua[1] === 'Edg' ? 'Edge' : ua[1]} ${ua[2]}` : 'unknown' },
canvasSelfTest: d.canvasSelfTest, settings,
references: { total: seeds.length, byPicture: seeds.filter(isPicture).length, exactFileOnly: seeds.filter(isMd5Only).length,
withDetail: seeds.filter(s => s.detail).length, withMainPicture: seeds.filter(s => s.part).length, withPieces: seeds.filter(s => s.pieces).length,
categories: cats.length, falsePositives: allow.length },
page: { kind: d.page.kind, stats: d.page.stats, browser: d.page.browser, memory: d.page.memory, matched: d.page.matched.length,
nearMisses: d.page.nearMisses.length, pending: d.page.pending.length, errors: d.page.errors.map(e => scrub(e.error)) },
sessions: d.sessions.map(s => ({ page: kindOf(s.url), control: s.control, stats: s.stats })),
logs: { byType, scriptErrors: d.logs.filter(l => l.type === 'script-error').slice(-20).map(l => ({ where: l.where, error: scrub(l.error), stack: scrub(l.stack) })) },
};
}
const ARCHIVES_URL = 'https://4chenz.github.io/archives.json/archives.json';
const ARCHIVES_MAX_AGE = 7 * 24 * 3600 * 1000;
let archivesTried = false, archivesError = null;
/**
* Refresh the archive table from the list 4chan XT uses. Unlike XT (every 2 days on page load),
* this only runs when the settings panel is opened, at most weekly, or on demand.
* Only the compact table (~2 KB) is stored. Resolves true if the table changed.
*/
async function refreshArchives(force) {
const age = archives.updated ? Date.now() - Date.parse(archives.updated) : Infinity;
if (!force && (!cfg.archiveUpdates || archivesTried || age < ARCHIVES_MAX_AGE)) return false;
archivesTried = true;
archivesError = null;
try {
const blob = await withTimeout(fetchBlob(ARCHIVES_URL), 20000, 'archive list request');
if (blob.size > 256 * 1024) throw new Error('archive list is unexpectedly large');
const table = buildArchiveTable(JSON.parse(await blob.text()));
const n = Object.keys(table.boards).length;
if (n < 20) throw new Error(`downloaded list covers only ${n} boards; keeping the current one`);
archives = { ...table, updated: new Date().toISOString() };
write('archives', archives);
log('archives-updated', { archives: table.hosts.length, boards: n });
return true;
} catch (e) {
archivesError = String(e && e.message || e);
log('archives-update-failed', { error: archivesError });
return false;
}
}
/**
* Source cell in the reference list: where it was blacklisted (as text) plus archive links. No link to
* the 4chan post itself: references outlast their threads, so the archived copy is the one that works.
*/
function sourceLinks(s) {
if (!s.board) return el('span', { className: 'mbgx-muted', title: 'Imported without a source board' }, 'imported');
const link = (href, text, title) => el('a', { href, target: '_blank', rel: 'noopener noreferrer', title }, text);
const arch = archiveLinks(s, archives, cfg.archivePick[s.board]);
const where = s.post ? `/${s.board}/ No.${s.post}` : `/${s.board}/`;
return [
el('span', { title: s.post ? 'Where it was blacklisted (see its archived copy)' : 'Board it was blacklisted on (no post number kept)' }, where),
arch && arch.post && [' · ', link(arch.post, 'archive', `Archived post on ${arch.host}`)],
arch && arch.search && [' · ', link(arch.search, 'find image', `Search ${arch.searchHost} for every post using this exact file (MD5)`)],
];
}
/** Number inputs for a set of matching rules (general settings or a category's own). */
function ruleFields(r, prefix) {
const num = (key, label) => el('label', {}, `${label} `, el('input', { type: 'number', id: `${prefix}-${key}`, className: `mbgx-rule-${key}`, min: 0, max: RULE_LIMIT, value: r[key] }));
return el('span', { className: 'mbgx-rules' }, num('a', 'aHash max distance'), ' ', num('d', 'dHash max distance'), ' ', num('margin', 'near-miss margin'), ' ',
el('label', {}, 'Mode ', el('select', { id: `${prefix}-mode`, className: 'mbgx-rule-mode' },
el('option', { value: 'both', selected: r.mode !== 'either' }, 'Both hashes must match (recommended)'),
el('option', { value: 'either', selected: r.mode === 'either' }, 'Either hash matches (aggressive)'))));
}
const readRules = box => cleanRules({ a: box.querySelector('.mbgx-rule-a').value, d: box.querySelector('.mbgx-rule-d').value,
margin: box.querySelector('.mbgx-rule-margin').value, mode: box.querySelector('.mbgx-rule-mode').value });
const TABS = [['general', 'General'], ['detection', 'Detection'], ['references', 'References'], ['falsePositives', 'False positives'], ['diagnostics', 'Diagnostics']];
const PAGE_SIZE = 100;
/**
* The settings panel. Tabs are built when opened (only the open tab exists in the page), the
* reference and false-positive lists show one page of 100 rows, and checkboxes use one delegated
* listener, so it stays quick with tens of thousands of references.
*/
function panel() {
if (document.getElementById('mbgx-panel')) return;
let tab = 'general', timer = 0;
try { tab = sessionStorage.getItem('mbgx-tab') || 'general'; } catch (e) { /* storage blocked: start on General */ }
if (!TABS.some(([k]) => k === tab)) tab = 'general';
const close = () => {
clearInterval(timer);
document.removeEventListener('keydown', onKey, true);
backdrop.remove();
p.remove();
};
const onKey = e => {
if (e.key !== 'Escape') return;
e.stopPropagation();
close();
};
const backdrop = el('div', { className: 'mbgx-backdrop', onclick: close });
const tabBar = el('div', { className: 'mbgx-tabs', role: 'tablist' });
const body = el('div', { className: 'mbgx-tab-body' });
guard('control', applyTheme);
const p = el('div', { id: 'mbgx-panel', className: 'mbgx-panel' },
el('h3', {}, `Better Filter v${VERSION}`), tabBar, body,
el('p', {}, el('button', { type: 'button', onclick: close }, 'Close')));
let update = null; // the open tab's live refresh, called every second
// Jump to a filtered post; with stubs off there is nothing to scroll to, so reveal it first.
const goTo = st => {
close();
if (st.result && st.result.match && !stubsFor(st.result.match) && !st.revealed) revealPost(st);
st.target.scrollIntoView({ block: 'center' });
st.target.classList.add('mbgx-flash');
setTimeout(() => st.target.classList.remove('mbgx-flash'), 1600);
};
const goLink = st => el('a', { href: 'javascript:;', className: 'mbgx-goto', title: 'Go to this post', onclick: () => goTo(st) }, `No.${st.no}`);
const saveSettings = patch => {
if ('workers' in patch && patch.workers !== cfg.workers) setWorkers(patch.workers);
cfg = { ...cfg, ...patch };
write('settings', cfg);
log('settings-saved', { settings: cfg });
for (const st of posts.values()) st.logged = {};
applyStubMode();
rematchAll();
};
const check = (id, key, label) => el('label', {}, el('input', { type: 'checkbox', id, checked: !!cfg[key], onchange: e => saveSettings({ [key]: e.target.checked }) }), ` ${label}`);
const renderTabs = () => tabBar.replaceChildren(...TABS.map(([k, label]) => {
const n = k === 'references' ? ` (${seeds.length})` : k === 'falsePositives' ? ` (${allow.length})`
: k === 'diagnostics' && (stats.hashErrors || stats.scriptErrors) ? ' (!)' : '';
return el('button', { type: 'button', className: 'mbgx-tab', role: 'tab', 'aria-selected': String(k === tab), dataset: { tab: k }, onclick: () => show(k) }, label + n);
}));
const show = k => {
tab = k;
try { sessionStorage.setItem('mbgx-tab', k); } catch (e) { /* not remembered */ }
update = null;
body.replaceChildren(BUILD[k]());
renderTabs();
if (update) update();
};
// ------------------------------------------------------------ General
const filteredList = () => {
const box = el('fieldset', { className: 'mbgx-filtered-list' });
let sig = null;
const render = () => {
const rows = cfg.enabled ? [...posts.values()].filter(st => st.result && st.result.match && st.role !== 'op' && st.root && st.root.isConnected) : [];
const s = rows.map(st => `${st.key}:${st.result.match.seed.id}:${st.revealed}:${nativeHidden(st.target)}`).join() + '|' + document.querySelectorAll('.mbgx-by-reply').length;
if (s === sig) return;
sig = s;
const hidden = rows.filter(st => !st.revealed).length;
box.replaceChildren(el('legend', {}, `Filtered on this page (${hidden} hidden${rows.length > hidden ? `, ${rows.length - hidden} revealed` : ''})`),
rows.length ? el('div', { className: 'mbgx-scroll' }, el('table', {},
el('tr', {}, ['Post', 'Reference', 'Category', 'How', 'aHash', 'dHash', 'State', ''].map(t => el('th', {}, t))),
rows.map(st => {
const m = st.result.match, c = catById(m.seed.cat);
return el('tr', { dataset: { mbgxKey: st.key } },
el('td', {}, goLink(st), st.role === 'reply' ? '' : el('span', { className: 'mbgx-muted' }, st.role === 'thread' ? ' (thread)' : ' (catalog)')),
el('td', {}, m.seed.id), el('td', {}, c ? c.name : '—'), el('td', { className: 'mbgx-how' }, methodOf(m)),
el('td', {}, m.da == null ? '–' : `${m.da}/64`), el('td', {}, m.dd == null ? '–' : `${m.dd}/64`),
el('td', {}, st.revealed ? 'revealed' : nativeHidden(st.target) ? `hidden (also by ${nativeName()})` : 'hidden'),
el('td', {}, el('button', { type: 'button', className: 'mbgx-row-toggle', onclick: () => {
if (st.revealed) rehidePost(st); else revealPost(st);
render();
} }, st.revealed ? 'Hide again' : 'Reveal')));
}))) : el('p', { className: 'mbgx-muted' }, 'Nothing filtered on this page.'),
el('p', { className: 'mbgx-muted mbgx-reply-count' }, ''));
const n = document.querySelectorAll('.mbgx-by-reply').length;
box.querySelector('.mbgx-reply-count').textContent = n ? `Also hidden: ${n} repl${n === 1 ? 'y' : 'ies'} to filtered posts (each has its own Reveal).` : '';
};
return [box, render];
};
const BUILD = {};
BUILD.general = () => {
const [box, render] = filteredList();
update = render;
return el('div', {}, box,
el('fieldset', {}, el('legend', {}, 'Filtering'),
check('mbgx-on', 'enabled', 'Enable automatic filtering'), el('br'),
check('mbgx-stubs', 'stubs', 'Show a stub for filtered posts (off: remove them from view completely; turn back on to reveal one)'), el('br'),
check('mbgx-replies', 'hideReplies', 'Also hide replies to filtered posts, and replies to those (like 4chan XT\'s "recursive" filter option; off in 4chan and XT by default)')),
el('p', { className: 'mbgx-muted' }, 'Replies with a matching image are filtered; an OP filters its whole thread on board pages and its tile in the catalog. OPs are never filtered on their own thread page. ' +
'A category can have its own settings for these and for the 4chan X options (References: choose the category → Settings).'),
el('fieldset', {}, el('legend', {}, 'Appearance'),
el('label', {}, 'Window colours ', el('select', { id: 'mbgx-theme-pick', onchange: e => {
cfg = { ...cfg, theme: e.target.value };
write('settings', cfg);
applyTheme();
} }, [['page', 'Match the page (its 4chan style, or your 4chan X / StyleChan / OneeChan theme)'], ['dark', 'Dark'], ['light', 'Light']]
.map(([v, t]) => el('option', { value: v, selected: (cfg.theme || 'page') === v }, t))))),
xBox());
};
// 4chan X / XT / XZ / neXT: where blacklisted MD5s go, the opt-in auto-add, and explicit import.
const xBox = () => {
const fam = xFamily();
const box = el('fieldset', { className: 'mbgx-xfam' }, el('legend', {}, '4chan X, XT, XZ and neXT'));
if (!fam) {
box.append(el('p', { className: 'mbgx-muted' }, 'None is running on this page. Better Filter works without them; with one running, ' +
'blacklisted files can also be added to its MD5 filter, and its MD5 filter can be imported here.'));
return box;
}
const dest = el('input', { type: 'checkbox', id: 'mbgx-md5dest', checked: cfg.md5Dest === 'both', onchange: e => saveSettings({ md5Dest: e.target.checked ? 'both' : 'native' }) });
const status = el('span', { className: 'mbgx-muted mbgx-xf-status' });
box.append(
el('p', {}, `Detected: ${fam.name}${fam.version ? ` ${fam.version}` : ''}` + (fam.tainted ? ' — two copies are enabled (it reports this itself); adding to its filter is disabled' : '')),
el('label', {}, dest, ` Blacklisting a file also adds it to ${fam.name}'s MD5 filter (can be changed each time)`), el('br'),
check('mbgx-xauto', 'xAuto', `Also add files Better Filter catches on its own to ${fam.name}'s MD5 filter (a mistake then stays there until you remove it)`), el('br'),
check('mbgx-xnostub', 'xNoStub', `Lines added to ${fam.name} hide posts without a stub`), el('br'),
el('button', { type: 'button', className: 'mbgx-xf-read', onclick: async e => {
e.target.disabled = true;
status.textContent = ' reading…';
try {
const r = await xReadMD5();
status.textContent = '';
const msg = await importXDialog(r.text, `${r.name}${r.version ? ` ${r.version}` : ''}`);
if (msg) notify(msg);
} catch (err) {
status.textContent = ` ${err && err.message || err}`;
} finally { e.target.disabled = false; }
} }, `Import ${fam.name}'s MD5 filter…`), status,
el('p', { className: 'mbgx-muted' }, `Better Filter only adds lines to ${fam.name}'s filter and never removes any: removing a reference here leaves its line there.` + (xAutoError ? ` Automatic adding stopped on this page: ${xAutoError}.` : '') +
(stats.xAutoAdded ? ` Added automatically on this page: ${stats.xAutoAdded}.` : '')));
return box;
};
// ------------------------------------------------------------ Detection
BUILD.detection = () => {
const rules = ruleFields(cfg, 'mbgx');
rules.addEventListener('change', () => saveSettings(readRules(rules)));
const near = el('fieldset', { className: 'mbgx-near-list' });
let sig = null;
// Why a near miss was not filtered, in plain words, by the rules that reference is judged by.
const whyNot = r => r.low ? 'near-blank image (only references made from one match it)'
: r.detailVeto != null ? `within the limits, but its detail differs from the reference's (${Math.round(r.detailVeto * 100)}% alike)`
: [r.best.da > r.best.rule.a && `aHash ${r.best.da} > ${r.best.rule.a}`, r.best.dd > r.best.rule.d && `dHash ${r.best.dd} > ${r.best.rule.d}`].filter(Boolean).join(', ');
update = () => {
const rows = [...posts.values()].filter(st => st.result && st.result.near && st.root && st.root.isConnected);
const s = rows.map(st => { const b = st.result.best; return `${st.key}:${b ? `${b.seed.id}:${b.da}:${b.dd}` : ''}`; }).join() + `|${cfg.a},${cfg.d},${cfg.margin}`;
if (s === sig) return;
sig = s;
near.hidden = !rows.length;
near.replaceChildren(el('legend', {}, `Near misses on this page (${rows.length})`),
el('div', { className: 'mbgx-scroll' }, el('table', {},
el('tr', {}, ['Post', 'Closest reference', 'aHash', 'dHash', 'Not filtered because'].map(t => el('th', {}, t))),
rows.map(st => {
const b = st.result.best;
return el('tr', { dataset: { mbgxKey: st.key } }, el('td', {}, goLink(st)), el('td', {}, b.seed.id),
el('td', {}, `${b.da}/64`), el('td', {}, `${b.dd}/64`), el('td', {}, whyNot(st.result)));
}))));
};
return el('div', {},
el('fieldset', {}, el('legend', {}, 'General matching rules'), rules,
el('p', { className: 'mbgx-muted' }, 'Used by Uncategorised references and by categories without their own limits (References: choose the category → Settings). ' +
'A near miss is an image within the limits plus the margin that was not filtered.')),
md5Box(), extendedBox(), near);
};
const md5Box = () => {
const n = refDB.md5Seeds.length, only = seeds.filter(isMd5Only).length;
return el('fieldset', { className: 'mbgx-md5' }, el('legend', {}, 'Exact files'),
check('mbgx-md5match', 'md5Match', 'Match identical files by MD5 (nothing is downloaded)'), el('br'),
check('mbgx-complete-md5', 'completeMd5', 'When an exact-file reference hides a post, add that post\'s picture to it (then reposts and edited copies are caught too)'),
el('p', { className: 'mbgx-muted mbgx-md5-count' }, `${n} reference${n === 1 ? '' : 's'} with an MD5, ${only} of them exact-file only. ` +
'Changed copies of a file are found by picture instead. The native catalog page lists no MD5s. ' +
'References → "Look up in archive…" finds exact-file references\' pictures in a 4chan archive.'));
};
const extendedBox = () => {
const withDetail = seeds.filter(s => s.detail).length, withPieces = seeds.filter(s => s.pieces).length;
return el('fieldset', { className: 'mbgx-extended' }, el('legend', {}, 'Edited copies'),
check('mbgx-editedCopies', 'editedCopies', 'Also filter edited copies of blacklisted pictures (recommended)'), el('br'),
el('span', { className: 'mbgx-subopt' }, check('mbgx-pieces', 'pieces', 'Including heavily cropped copies (such as only half of the picture)')),
el('p', { className: 'mbgx-muted' }, 'Checked only for pictures with no ordinary match. Whitelist any mistake as usual. ' +
'A category can switch these, close variants and exact files on or off for its references (References: choose the category → Settings).'),
el('p', { className: 'mbgx-muted mbgx-detail-count' }, `${withDetail} of ${seeds.length} references have the extra detail that makes this more reliable, ` +
`and ${withPieces} the pieces that catch heavily cropped copies (both added automatically; a reference without them gets them when its exact image is seen again).`),
el('hr'),
check('mbgx-closeVariants', 'closeVariants', 'Also filter pictures that are only very similar to a blacklisted one'),
el('p', { className: 'mbgx-muted' }, 'Off by default. Can catch pictures you would call different; they show as "close variant" in "Filtered on this page", so you can whitelist them.'));
};
// ------------------------------------------------------------ References
const sel = new Set();
// the order of the references table is kept (in storage of its own: not a setting, never exported)
const SORTS = [['added', 'Newest first'], ['oldest', 'Oldest first'], ['id', 'By ID'], ['category', 'By category'], ['board', 'By source board']];
let catFilter = 'all', query = '', sortBy = read('refSort', 'added'), page = 0, jsonBox = null;
if (!SORTS.some(([v]) => v === sortBy)) sortBy = 'added';
BUILD.references = () => {
const root = el('div', { className: 'mbgx-refs' });
const render = () => {
seeds = loadSeeds();
cats = loadCats();
if (catFilter !== 'all' && catFilter !== '' && !catById(catFilter)) catFilter = 'all';
const names = new Map(cats.map(c => [c.id, c.name]));
const counts = new Map();
for (const s of seeds) counts.set(names.has(s.cat) ? s.cat : '', (counts.get(names.has(s.cat) ? s.cat : '') || 0) + 1);
const q = query.trim().toLowerCase();
let list = seeds.filter(s => (catFilter === 'all' || (catFilter === '' ? !names.has(s.cat) : s.cat === catFilter)) &&
(!q || [s.id, s.a, s.d, s.board, s.post, s.md5, names.get(s.cat), ...(s.tags || []), ...(s.boards || [])].some(v => v && String(v).toLowerCase().includes(q))));
const num = s => +(/\d+/.exec(s.id) || [0])[0];
if (sortBy === 'added') list = list.slice().reverse();
else if (sortBy === 'id') list = list.slice().sort((x, y) => num(x) - num(y));
else if (sortBy === 'category') list = list.slice().sort((x, y) => (names.get(x.cat) || '~').localeCompare(names.get(y.cat) || '~'));
else if (sortBy === 'board') list = list.slice().sort((x, y) => (x.board || '~').localeCompare(y.board || '~'));
for (const id of sel) if (!seeds.some(s => s.id === id)) sel.delete(id);
const pages = Math.max(1, Math.ceil(list.length / PAGE_SIZE));
page = Math.min(page, pages - 1);
const shown = list.slice(page * PAGE_SIZE, (page + 1) * PAGE_SIZE);
const cat = catById(catFilter);
const toolbar = el('div', { className: 'mbgx-toolbar' },
// Changing what is shown clears the selection, so an action never touches references you can't see.
el('select', { className: 'mbgx-cat-filter', onchange: e => { catFilter = e.target.value; page = 0; sel.clear(); render(); } },
el('option', { value: 'all', selected: catFilter === 'all' }, `All references (${seeds.length})`),
el('option', { value: '', selected: catFilter === '' }, `Uncategorised (${counts.get('') || 0})`),
cats.map(c => el('option', { value: c.id, selected: catFilter === c.id }, `${c.name} (${counts.get(c.id) || 0})${c.enabled === false ? ' – disabled' : ''}`))),
el('input', { type: 'search', className: 'mbgx-search', placeholder: 'Search ID, tag, board, post, hash…', value: query,
oninput: e => { query = e.target.value; page = 0; sel.clear(); clearTimeout(root.searchTimer); root.searchTimer = setTimeout(render, 200); } }),
el('select', { className: 'mbgx-sort', onchange: e => { sortBy = e.target.value; write('refSort', sortBy); render(); } },
SORTS.map(([v, t]) => el('option', { value: v, selected: sortBy === v }, t))),
menu('Categories ▾', [
['New category…', newCategory],
cat && [`Rename "${cat.name}"…`, () => renameCategory(cat)],
cat && ['Applies to (boards)…', () => categoryBoards(cat)],
cat && ['Settings (general and detection)…', () => categorySettings(cat)],
cat && [cat.enabled === false ? 'Enable category' : 'Disable category', () => { saveCats(cats.map(c => (c.id === cat.id ? { ...c, enabled: c.enabled === false } : c)), { toggled: cat.id }); render(); }],
cat && ['Merge into…', () => mergeCategory(cat)],
cat && ['Export this category…', () => exportDialog('references', [{ label: cat.name, list: seeds.filter(s => s.cat === cat.id) }])],
cat && ['Import into this category…', () => chooseImportFile(render, cat.id)],
cat && ['Delete category…', () => deleteCategory(cat)],
['Clear all references…', clearAllRefs],
], cat ? `New category, and everything for "${cat.name}": settings, boards, rename, merge, export, import, delete`
: 'New category; choose a category in the first list for its settings, boards, rename and more'));
const catBar = cat ? el('div', { className: 'mbgx-catbar' + (cat.enabled === false ? ' mbgx-disabled' : '') },
el('p', {}, el('b', {}, cat.name), ` · ${refCount(counts.get(cat.id) || 0)} · applies to ${scopeText(cat.boards)} · `,
cat.rules ? `own rules: ${ruleText(cat.rules)}` : 'general rules',
cat.settings ? ` · own settings: ${settingsText(cat.settings)}` : '', cat.enabled === false ? ' · disabled (not filtering)' : ''),
el('div', { className: 'mbgx-catbar-actions' },
el('button', { type: 'button', className: 'mbgx-cat-settings-btn', onclick: () => categorySettings(cat) }, 'Settings…'),
el('button', { type: 'button', className: 'mbgx-cat-boards-btn', onclick: () => categoryBoards(cat) }, 'Applies to…'),
el('button', { type: 'button', className: 'mbgx-cat-rename-btn', onclick: () => renameCategory(cat) }, 'Rename…'),
el('span', { className: 'mbgx-muted' }, ' More in Categories ▾')))
: el('p', { className: 'mbgx-muted mbgx-cat-hint' }, cats.length
? 'To change a category\'s settings, choose it in the first list, or click its name in the table.'
: 'Categories group references; each can have its own settings. Categories ▾ → New category… to make one.');
const selBar = sel.size > 0 && el('div', { className: 'mbgx-selbar' },
el('b', {}, `${sel.size} selected`),
list.length > sel.size && list.some(s => !sel.has(s.id)) && el('a', { href: 'javascript:;', className: 'mbgx-sel-all', onclick: () => { list.forEach(s => sel.add(s.id)); render(); } }, `Select all ${list.length} matching`),
el('a', { href: 'javascript:;', className: 'mbgx-sel-clear', onclick: () => { sel.clear(); render(); } }, 'Clear selection'),
el('select', { className: 'mbgx-move', onchange: e => { const v = e.target.value; e.target.value = 'x'; if (v !== 'x') moveSelected(v); } },
el('option', { value: 'x' }, 'Move to…'), el('option', { value: '' }, 'Uncategorised'),
cats.map(c => el('option', { value: c.id }, c.name)), el('option', { value: 'new' }, '+ New category…')),
el('button', { type: 'button', onclick: tagSelected }, 'Tags…'),
el('button', { type: 'button', onclick: boardsSelected }, 'Boards…'),
el('button', { type: 'button', onclick: () => exportDialog('references', [{ label: 'Selected', list: seeds.filter(s => sel.has(s.id)) }]) }, 'Export…'),
el('button', { type: 'button', className: 'mbgx-sel-delete', onclick: deleteSelected }, 'Delete'));
const allOnPage = shown.length && shown.every(s => sel.has(s.id));
const table = shown.length ? el('div', { className: 'mbgx-scroll' }, el('table', {},
el('tr', {}, el('th', {}, el('input', { type: 'checkbox', className: 'mbgx-sel-page', checked: allOnPage, title: 'Select this page' })),
['ID', 'Category', 'Tags', 'Source', 'Applies to', 'Added', ''].map(t => el('th', {}, t))),
shown.map(s => {
const e = effective(s), c = catById(s.cat);
const pic = isPicture(s);
return el('tr', { dataset: { mbgxSeed: s.id }, className: e.active ? '' : 'mbgx-disabled',
title: (pic ? `aHash ${s.a} dHash ${s.d}\nMatched with ${e.rulesFrom === 'category' ? 'its category\'s' : 'the general'} rules: ${ruleText(e.rules)}` : 'Matches this exact file only') + (s.md5 ? `\nMD5 ${s.md5}${cfg.md5Match ? '' : ' (exact-file matching is off)'}` : '') },
el('td', {}, el('input', { type: 'checkbox', className: 'mbgx-sel', checked: sel.has(s.id), dataset: { id: s.id } })),
el('td', {}, s.id + (pic ? '' : ' (exact file)'), ' ',
pic && (s.pieces ? el('span', { className: 'mbgx-tag mbgx-has-pieces', title: `${pieceCount(s.pieces)} pieces: heavily cropped copies of this picture are caught too` }, 'pieces')
: el('span', { className: 'mbgx-tag mbgx-no-pieces', title: 'No pieces yet, so heavily cropped copies are not caught. Added when this exact image is seen again, or by importing a file that has them.' }, 'no pieces'))),
el('td', {}, c ? el('a', { href: 'javascript:;', className: 'mbgx-cat-link', title: `Show "${c.name}" with its settings`,
onclick: () => { catFilter = c.id; page = 0; sel.clear(); render(); } }, c.name) : el('span', { className: 'mbgx-muted' }, '—')),
el('td', { className: 'mbgx-tags-cell' }, (s.tags || []).slice(0, TAGS_SHOWN).map(t => el('button', { type: 'button', className: 'mbgx-tag mbgx-tag-edit', title: `Edit, rename or remove "${t}", or add tags`,
onclick: () => editTags(s.id, t) }, t)),
(s.tags || []).length > TAGS_SHOWN && el('button', { type: 'button', className: 'mbgx-tag mbgx-tag-more', title: (s.tags || []).slice(TAGS_SHOWN).join('\n'),
onclick: () => editTags(s.id) }, `+${s.tags.length - TAGS_SHOWN} more`),
el('button', { type: 'button', className: 'mbgx-tag mbgx-tag-plus', title: 'Add or edit this reference\'s tags', 'aria-label': 'Add or edit tags', onclick: () => editTags(s.id) }, '+')),
el('td', {}, sourceLinks(s)),
el('td', {}, el('input', { type: 'text', className: 'mbgx-scope', value: boardsText(s.boards), placeholder: c && c.boards ? `as category: ${boardsText(c.boards)}` : 'all boards',
title: 'Boards this reference filters on, separated by commas, e.g. "v, vg, a" (spaces are ignored). Empty: its category\'s boards, or all boards.',
onchange: ev => { if (setSeedBoards(s.id, ev.target.value)) render(); else ev.target.value = boardsText(s.boards); } })),
el('td', {}, (s.added || '').slice(0, 10)),
el('td', {}, el('button', { type: 'button', className: 'mbgx-ref-remove', title: 'Remove this reference', onclick: () => removeSeed(s.id).then(ok => ok && render()) }, '×')));
}))) : el('p', { className: 'mbgx-muted' }, seeds.length ? 'No references match.' : 'No references yet. Use "⊘ blacklist" on a post, or import a blacklist file.');
table.addEventListener('change', ev => {
const t = ev.target;
if (t.matches('.mbgx-sel')) { if (t.checked) sel.add(t.dataset.id); else sel.delete(t.dataset.id); render(); }
else if (t.matches('.mbgx-sel-page')) { shown.forEach(s => (t.checked ? sel.add(s.id) : sel.delete(s.id))); render(); }
});
const pager = pages > 1 && el('p', { className: 'mbgx-pager' },
el('button', { type: 'button', className: 'mbgx-prev', disabled: page === 0, onclick: () => { page--; render(); } }, '‹ Previous'),
` Page ${page + 1} of ${pages} (${list.length} references) `,
el('button', { type: 'button', className: 'mbgx-next', disabled: page >= pages - 1, onclick: () => { page++; render(); } }, 'Next ›'));
const typing = keepTyping(root);
root.replaceChildren(
el('fieldset', {}, el('legend', {}, `References (${seeds.length})`), toolbar, catBar, selBar, table, pager,
el('p', {},
// Preselected: the selection if there is one, else what is shown, else everything.
el('button', { type: 'button', onclick: () => exportDialog('references', [
sel.size && { label: 'Selected', list: seeds.filter(s => sel.has(s.id)) },
q && { label: 'Matching the search', list },
cat && { label: `Category: ${cat.name}`, list: seeds.filter(s => s.cat === cat.id) },
catFilter === '' && { label: 'Uncategorised', list: seeds.filter(s => !names.has(s.cat)) },
{ label: 'All references', list: seeds },
].filter(Boolean)) }, 'Export blacklist…'),
el('button', { type: 'button', onclick: () => chooseImportFile(render, cat ? cat.id : undefined) }, 'Import blacklist file…'),
el('button', { type: 'button', className: 'mbgx-xf-paste', onclick: () => importXDialog(null).then(msg => { if (msg) notify(msg); render(); }) }, 'Import 4chan X MD5 filter…'),
el('button', { type: 'button', className: 'mbgx-lookup-btn', title: 'Find the selected references (or those shown) in a 4chan archive by their MD5, ' +
'to add their picture and pieces: exact-file references then catch reposts and edited copies too',
onclick: () => lookupDialog((sel.size ? seeds.filter(s => sel.has(s.id)) : list).map(s => s.id)) }, 'Look up in archive…'),
el('span', { className: 'mbgx-muted' }, ' Import merges with your list; duplicates are skipped.')),
jsonBox = jsonEditor(render)),
archivesBox());
typing();
renderTabs();
};
// ---- category and selection actions (all one storage write each)
const newCategory = async () => {
const name = el('input', { type: 'text', className: 'mbgx-cat-name', maxLength: 60 });
if (!await ask('New category', 'Create', 'Cancel', el('div', { className: 'mbgx-form' }, el('label', {}, 'Name ', name)))) return;
const [c, list] = categoryNamed(name.value);
saveCats(list, { created: c.id });
catFilter = c.id;
render();
};
const renameCategory = async cat => {
const name = el('input', { type: 'text', className: 'mbgx-cat-name', value: cat.name, maxLength: 60 });
if (!await ask(`Rename "${cat.name}"`, 'Rename', 'Cancel', el('div', { className: 'mbgx-form' }, el('label', {}, 'Name ', name)))) return;
if (cats.some(c => c.id !== cat.id && c.name.toLowerCase() === name.value.trim().toLowerCase())) { notify('Another category already has that name. Use "Merge into…" to combine them.'); return; }
saveCats(cats.map(c => (c.id === cat.id ? { ...c, name: name.value } : c)), { renamed: cat.id });
render();
};
const categoryBoards = async cat => {
const box = el('input', { type: 'text', className: 'mbgx-cat-boards', value: boardsText(cat.boards), placeholder: 'all boards' });
if (!await ask(`Where "${cat.name}" filters\n\nBoards separated by commas, e.g. "vg, c", or empty for all boards. References with their own boards keep them.`,
'Save', 'Cancel', el('div', { className: 'mbgx-form' }, el('label', {}, 'Applies to ', box)))) return;
const list = parseBoards(box.value);
if (!list) return;
saveCats(cats.map(c => (c.id === cat.id ? { ...c, boards: list } : c)), { boards: cat.id });
render();
};
// A category's own general and detection settings: each option the same as the general one, or on / off.
const categorySettings = async cat => {
const own = el('select', { className: 'mbgx-cat-ruletype' }, el('option', { value: 'general', selected: !cat.rules }, 'Same as the general limits'),
el('option', { value: 'own', selected: !!cat.rules }, 'Its own limits'));
const fields = ruleFields(cat.rules || cfg, 'mbgx-cat');
const sync = () => { fields.hidden = own.value !== 'own'; };
own.addEventListener('change', sync);
sync();
const fam = xFamily(), set = cat.settings || {};
const row = ([k, , label, general]) => el('label', {}, label, el('select', { className: `mbgx-cat-set mbgx-cat-set-${k}`, dataset: { key: k } },
el('option', { value: 'general', selected: typeof set[k] !== 'boolean' }, `Same as general (now ${general() ? 'on' : 'off'})`),
el('option', { value: 'on', selected: set[k] === true }, 'On'),
el('option', { value: 'off', selected: set[k] === false }, 'Off')));
const group = g => CAT_SETTINGS.filter(([, x]) => x === g).map(row);
const showX = fam || LOCAL_SETTINGS.some(k => typeof set[k] === 'boolean');
const form = el('div', { className: 'mbgx-form mbgx-cat-settings' },
el('fieldset', {}, el('legend', {}, 'General'), group('general'),
showX && el('p', { className: 'mbgx-muted' }, `${fam ? fam.name : '4chan X, XT, XZ or neXT'} (kept in this browser only, never exported):`), showX && group('x')),
el('fieldset', {}, el('legend', {}, 'Detection'), el('label', {}, 'Matching limits ', own), fields, group('detection')));
if (!await ask(`Settings for "${cat.name}"\n\nFor posts its references filter. Anything left "Same as general" follows the General and Detection tabs. ` +
'Lower distances are stricter (fewer false positives); "either hash" is the most aggressive.', 'Save', 'Cancel', form)) return;
const next = {};
for (const sel of form.querySelectorAll('.mbgx-cat-set')) if (sel.value !== 'general') next[sel.dataset.key] = sel.value === 'on';
saveCats(cats.map(c => {
if (c.id !== cat.id) return c;
const { settings, ...rest } = c;
return { ...rest, rules: own.value === 'own' ? readRules(fields) : null, ...(Object.keys(next).length ? { settings: next } : {}) };
}), { settings: cat.id });
render();
};
const mergeCategory = async cat => {
const into = el('select', { className: 'mbgx-merge-into' }, el('option', { value: '' }, 'Uncategorised'),
cats.filter(c => c.id !== cat.id).map(c => el('option', { value: c.id }, c.name)));
const n = seeds.filter(s => s.cat === cat.id).length;
if (!await ask(`Merge "${cat.name}" into another category?\n\nIts ${refCount(n)} move there and take on that category's boards and rules (unless they have their own boards). "${cat.name}" is then deleted.`,
'Merge', 'Cancel', el('div', { className: 'mbgx-form' }, el('label', {}, 'Into ', into)))) return;
write('seeds', loadSeeds().map(s => (s.cat === cat.id ? cleanSeed({ ...s, cat: into.value || undefined }) : s)));
saveCats(cats.filter(c => c.id !== cat.id), { merged: cat.id, into: into.value });
catFilter = into.value || 'all';
render();
};
const deleteCategory = async cat => {
const n = seeds.filter(s => s.cat === cat.id).length;
const what = el('select', { className: 'mbgx-delete-how' },
el('option', { value: 'keep' }, `Keep its ${refCount(n)} (they become Uncategorised${cat.boards ? ' and apply to all boards unless they have their own' : ''})`),
el('option', { value: 'all' }, `Delete its ${refCount(n)} too`));
if (!await ask(`Delete the category "${cat.name}"?`, 'Delete', 'Cancel', el('div', { className: 'mbgx-form' }, what))) return;
if (what.value === 'all' && n && !await ask(`Delete ${n} reference${n === 1 ? '' : 's'} permanently?\n\nThis cannot be undone (export the category first if unsure).`, `Delete ${n}`)) return;
write('seeds', what.value === 'all' ? loadSeeds().filter(s => s.cat !== cat.id) : loadSeeds().map(s => (s.cat === cat.id ? cleanSeed({ ...s, cat: undefined }) : s)));
saveCats(cats.filter(c => c.id !== cat.id), { deleted: cat.id, references: what.value === 'all' ? n : 0 });
catFilter = 'all';
render();
};
const clearAllRefs = async () => {
const n = loadSeeds().length;
if (!n) { notify('There are no references to clear.'); return; }
if (!await ask(`Clear all ${refCount(n)}?\n\nEvery blacklisted image is removed and posts hidden by them reappear. Categories, false positives and settings are kept. This cannot be undone (export the blacklist first if unsure).`, `Clear ${n} references`)) return;
sel.clear();
editSeeds(() => [], { cleared: n });
render();
};
const moveSelected = async v => {
let id = v;
if (v === 'new') {
const name = el('input', { type: 'text', className: 'mbgx-cat-name', maxLength: 60 });
if (!await ask(`Move ${sel.size} references to a new category`, 'Move', 'Cancel', el('div', { className: 'mbgx-form' }, el('label', {}, 'Name ', name)))) return;
const [c, list] = categoryNamed(name.value);
write('categories', list);
id = c.id;
}
editSeeds(list => list.map(s => (sel.has(s.id) ? cleanSeed({ ...s, cat: id || undefined }) : s)), { moved: sel.size, to: id });
sel.clear();
render();
};
const tagSelected = async () => {
const add = el('input', { type: 'text', className: 'mbgx-tags-add', placeholder: 'separated by commas' });
const counts = new Map();
for (const s of seeds) if (sel.has(s.id)) for (const t of s.tags || []) counts.set(t, (counts.get(t) || 0) + 1);
const present = [...counts].sort((x, y) => y[1] - x[1] || x[0].localeCompare(y[0]));
const remove = el('div', { className: 'mbgx-tag-list' }, present.length ? present.map(([t, n]) =>
el('label', {}, el('input', { type: 'checkbox', className: 'mbgx-tags-remove', value: t }), ' ', el('span', { className: 'mbgx-tag' }, t), ` ${n}`))
: el('span', { className: 'mbgx-muted' }, 'none of the selected references have tags'));
if (!await ask(`Tags for ${refCount(sel.size)}`, 'Apply', 'Cancel', el('div', { className: 'mbgx-form' }, el('label', {}, 'Add ', add),
el('div', {}, 'Remove (ticked):', remove)))) return;
const plus = parseTags(add.value), minus = new Set([...remove.querySelectorAll('.mbgx-tags-remove:checked')].map(c => c.value));
const full = seeds.filter(s => sel.has(s.id) && tagsOver([...(s.tags || []), ...plus].filter(t => !minus.has(t)))).length;
editSeeds(list => list.map(s => (sel.has(s.id) ? cleanSeed({ ...s, tags: [...(s.tags || []), ...plus].filter(t => !minus.has(t)) }) : s)), { tagged: sel.size });
render();
if (full) notify(`A reference keeps at most ${TAG_LIMIT} tags: ${full} of them could not take all the new ones.`);
};
// One reference's tags: rename, remove or add; renames and removals can also go to every
// other reference with that tag.
const editTags = async (id, focus) => {
const s = seeds.find(x => x.id === id);
if (!s) return;
const own = s.tags || [];
const rows = own.map(t => {
const name = el('input', { type: 'text', className: 'mbgx-tag-name', value: t, maxLength: 40, dataset: { was: t } });
const del = el('input', { type: 'checkbox', className: 'mbgx-tag-del' });
del.addEventListener('change', () => { name.disabled = del.checked; });
return el('div', { className: 'mbgx-tag-row' }, name, el('label', {}, del, ' Remove'));
});
const add = el('input', { type: 'text', className: 'mbgx-tag-new', placeholder: 'separated by commas' });
const others = seeds.filter(x => x.id !== id && (x.tags || []).some(t => own.includes(t))).length;
const everywhere = el('input', { type: 'checkbox', className: 'mbgx-tag-everywhere' });
const form = el('div', { className: 'mbgx-form mbgx-tag-editor' },
el('p', { className: 'mbgx-muted' }, `${own.length} of at most ${TAG_LIMIT} tags, each up to ${TAG_CHARS} characters.`),
rows.length ? el('div', { className: 'mbgx-tag-rows' }, rows) : el('p', { className: 'mbgx-muted' }, 'No tags yet.'),
el('label', {}, 'Add ', add),
others ? el('label', {}, everywhere, ` Also rename and remove these tags on the ${others} other reference${others === 1 ? '' : 's'} that have them`) : '');
setTimeout(() => {
const f = focus !== undefined ? rows.map(r => r.querySelector('.mbgx-tag-name')).find(x => x.dataset.was === focus) : add;
if (f) { f.focus(); if (f.select) f.select(); }
}, 0);
if (!await ask(`Tags of reference ${id}`, 'Save', 'Cancel', form)) return;
// old tag -> new name ('' = removed)
const change = new Map(rows.map(r => {
const name = r.querySelector('.mbgx-tag-name'), gone = r.querySelector('.mbgx-tag-del').checked;
return [name.dataset.was, gone ? '' : name.value.trim()];
}));
const rename = list => list.map(t => (change.has(t) ? change.get(t) : t)).filter(Boolean);
const plus = parseTags(add.value), all = everywhere.checked, over = tagsOver([...rename(own), ...plus]);
editSeeds(list => list.map(x => (x.id === id ? cleanSeed({ ...x, tags: [...rename(x.tags || []), ...plus] })
: all && (x.tags || []).some(t => change.has(t)) ? cleanSeed({ ...x, tags: rename(x.tags) }) : x)), { tagsEdited: id, everywhere: all });
render();
if (over) notify(`A reference keeps at most ${TAG_LIMIT} tags: ${over} of the new ones ${over === 1 ? 'was' : 'were'} not added.`);
};
const boardsSelected = async () => {
const box = el('input', { type: 'text', className: 'mbgx-sel-boards', placeholder: 'empty: their category\'s boards, or all boards' });
if (!await ask(`Where ${sel.size} references filter\n\nBoards separated by commas, e.g. "vg, c". Empty removes their own boards, so they follow their category (or apply everywhere).`,
'Apply', 'Cancel', el('div', { className: 'mbgx-form' }, el('label', {}, 'Applies to ', box)))) return;
const list = parseBoards(box.value);
if (!list) return;
editSeeds(all => all.map(s => (sel.has(s.id) ? cleanSeed({ ...s, boards: list }) : s)), { boards: list, count: sel.size });
render();
};
const deleteSelected = async () => {
const n = sel.size;
if (!await ask(`Delete ${n} selected reference${n === 1 ? '' : 's'}?\n\nPosts hidden only because of them reappear. This cannot be undone (export them first if unsure).`, `Delete ${n}`)) return;
editSeeds(list => list.filter(s => !sel.has(s.id)), { removed: n });
sel.clear();
render();
};
render();
return root;
};
const jsonEditor = done => {
const text = el('textarea', { spellcheck: false });
const box = el('details', { ontoggle: () => { if (box.open) text.value = JSON.stringify(loadSeeds(), null, 1); } },
el('summary', {}, 'Edit references as JSON (advanced)'), text,
el('button', { type: 'button', onclick: () => {
try {
const data = JSON.parse(text.value);
if (!Array.isArray(data) || !data.every(x => x && typeof x.id === 'string' && ((HEX64.test(x.a) && HEX64.test(x.d)) || normMd5(x.md5)))) throw new Error('Each reference needs a string "id" and either 16-digit lowercase hex "a" and "d", or an "md5".');
if (new Set(data.map(x => x.id)).size !== data.length) throw new Error('Reference IDs must be unique.');
if (!data.every(x => SEED_ID.test(x.id))) throw new Error('Reference IDs may only use letters, digits, "_" and "-" (up to 40).');
editSeeds(() => data.map(cleanSeed), { replacedWith: data.length });
done();
} catch (e) { notify(String(e && e.message || e)); }
} }, 'Save edited references'));
return box;
};
const archivesBox = () => {
const status = () => {
const upd = el('a', { href: 'javascript:;', onclick: () => {
upd.textContent = 'updating…';
refreshArchives(true).then(() => { if (tab === 'references') show('references'); });
} }, 'update now');
return el('p', { className: 'mbgx-muted mbgx-archives' },
`Archive links: ${archives.updated ? 'list downloaded ' + archives.updated.slice(0, 10) : 'built-in list (2026-10-28)'}` +
` · ${Object.keys(archives.boards).length} boards · `, upd,
archivesError && el('span', { className: 'mbgx-error' }, ` · update failed: ${archivesError}`));
};
// Per-board archive choice, like 4chan XT's archive settings. Kept open across re-renders.
const chooser = el('details', { className: 'mbgx-archive-chooser', open: !!panel.chooserOpen, ontoggle: () => { panel.chooserOpen = chooser.open; } });
const renderChooser = () => {
const boards = Object.keys(archives.boards).sort();
if (!boards.includes(panel.chooserBoard)) panel.chooserBoard = boards.includes(BOARD) ? BOARD : boards.includes('vg') ? 'vg' : boards[0];
const b = panel.chooserBoard, cands = archives.boards[b] || [], pick = cfg.archivePick[b] || {};
const auto = pickArchives(archives, b, null), host = i => archives.hosts[i];
const label = ([i, f]) => host(i) + ((f & 1) ? '' : ' (text only)');
const save = (kind, value) => {
const next = { ...(cfg.archivePick[b] || {}) };
if (value) next[kind] = value; else delete next[kind];
cfg.archivePick = { ...cfg.archivePick };
if (Object.keys(next).length) cfg.archivePick[b] = next; else delete cfg.archivePick[b];
write('settings', cfg);
show('references');
};
const used = pickArchives(archives, b, pick);
const select = (kind, autoText) => el('select', { className: `mbgx-pick-${kind}`, onchange: e => save(kind, e.target.value) },
el('option', { value: '', selected: !pick[kind] }, autoText),
cands.map(c => el('option', { value: host(c[0]), selected: pick[kind] === host(c[0]) }, label(c))));
const custom = Object.entries(cfg.archivePick).map(([bd, pk]) => `/${bd}/ ` + [pk.post && `view: ${pk.post}`, pk.search && `search: ${pk.search}`].filter(Boolean).join(', '));
chooser.replaceChildren(
el('summary', {}, 'Choose archives per board' + (custom.length ? ` (${custom.length} custom)` : '')),
el('p', {},
el('label', {}, 'Board ', el('select', { className: 'mbgx-pick-board', onchange: e => { panel.chooserBoard = e.target.value; renderChooser(); } },
boards.map(x => el('option', { value: x, selected: x === b }, `/${x}/`)))), ' ',
el('label', {}, 'View posts on ', select('post', `Automatic (${host(auto.post)})`)), ' ',
el('label', {}, 'Image search on ', select('search', `Same as viewing (${host(used.post)})`)), ' ',
(pick.post || pick.search) ? el('button', { type: 'button', onclick: () => { delete cfg.archivePick[b]; cfg.archivePick = { ...cfg.archivePick }; write('settings', cfg); show('references'); } }, 'Reset board') : ''),
el('p', { className: 'mbgx-muted' }, custom.length ? 'Custom: ' + custom.join(' · ') : 'All boards use the automatic choice: an archive that keeps images, when there is one; image search uses the same archive.'));
};
renderChooser();
return el('fieldset', {}, el('legend', {}, 'Archive links'), status(),
check('mbgx-archives', 'archiveUpdates', 'Keep archive links up to date (checks the 4chenz archive list weekly, only when this panel is opened)'), chooser);
};
// ------------------------------------------------------------ False positives
const fpSel = new Set();
let fpQuery = '', fpPage = 0;
BUILD.falsePositives = () => {
const root = el('fieldset', { className: 'mbgx-allow-list' });
const render = () => {
allow = loadAllow();
const keys = new Set(allow.map(fpKey));
for (const k of fpSel) if (!keys.has(k)) fpSel.delete(k);
const q = fpQuery.trim().toLowerCase();
const list = allow.slice().reverse().filter(x => !q || [x.post, x.seed, x.a, x.d, x.md5, x.thread].some(v => v && String(v).toLowerCase().includes(q)));
const pages = Math.max(1, Math.ceil(list.length / PAGE_SIZE));
fpPage = Math.min(fpPage, pages - 1);
const shown = list.slice(fpPage * PAGE_SIZE, (fpPage + 1) * PAGE_SIZE);
const postCell = x => {
const [board, no] = typeof x.post === 'string' && /^\w+\.\d+$/.test(x.post) ? x.post.split('.') : [];
if (!no) return '–';
const here = posts.get(x.post);
if (here && here.root && here.root.isConnected) return goLink(here);
const arch = archiveLinks({ board, thread: x.thread, post: no }, archives, cfg.archivePick[board]);
const link = (href, text) => el('a', { href, target: '_blank', rel: 'noopener noreferrer' }, text);
return [x.thread ? link(`https://boards.4chan.org/${board}/thread/${x.thread}#p${no}`, `/${board}/ No.${no}`) : `/${board}/ No.${no}`,
arch && arch.post && [' · ', link(arch.post, 'archive')]];
};
const removeKeys = async (ks, label) => {
if (!await ask(`${label}\n\nPosts with these images will be filtered again if they match a reference.`, 'Remove')) return;
guard('command', () => {
write('allow', loadAllow().filter(y => !ks.has(fpKey(y))));
log('false-positive-removed', { count: ks.size });
ks.forEach(k => fpSel.delete(k));
rematchAll();
render();
});
};
const table = shown.length ? el('div', { className: 'mbgx-scroll' }, el('table', {},
el('tr', {}, el('th', {}, el('input', { type: 'checkbox', className: 'mbgx-fp-sel-page', checked: shown.every(x => fpSel.has(fpKey(x))) })),
['Post', 'Had matched', 'aHash', 'dHash', 'Added', ''].map(t => el('th', {}, t))),
shown.map(x => el('tr', { className: 'mbgx-allow-row', title: [x.a && `aHash ${x.a} dHash ${x.d}`, x.md5 && `MD5 ${x.md5}`].filter(Boolean).join('\n') },
el('td', {}, el('input', { type: 'checkbox', className: 'mbgx-fp-sel', checked: fpSel.has(fpKey(x)), dataset: { key: fpKey(x) } })),
el('td', {}, postCell(x)),
el('td', {}, (x.seed || '–') + (x.a ? '' : ' (exact file)')),
el('td', {}, x.da != null ? `${x.da}/64` : '–'), el('td', {}, x.dd != null ? `${x.dd}/64` : '–'),
el('td', {}, (x.added || '').slice(0, 10)),
el('td', {}, el('button', { type: 'button', className: 'mbgx-allow-remove', onclick: () => removeKeys(new Set([fpKey(x)]), 'Remove this image from the false-positive list?') }, 'Remove')))))) :
el('p', { className: 'mbgx-muted' }, allow.length ? 'No false positives match.' : 'None. Use "False positive / whitelist" on a filtered post to add one.');
table.addEventListener('change', ev => {
const t = ev.target;
if (t.matches('.mbgx-fp-sel')) { if (t.checked) fpSel.add(t.dataset.key); else fpSel.delete(t.dataset.key); render(); }
else if (t.matches('.mbgx-fp-sel-page')) { shown.forEach(x => (t.checked ? fpSel.add(fpKey(x)) : fpSel.delete(fpKey(x)))); render(); }
});
const typing = keepTyping(root);
root.replaceChildren(el('legend', {}, `False positives (${allow.length})`),
allow.length > 10 ? el('div', { className: 'mbgx-toolbar' }, el('input', { type: 'search', className: 'mbgx-fp-search', placeholder: 'Search post, reference, hash…', value: fpQuery,
oninput: e => { fpQuery = e.target.value; fpPage = 0; clearTimeout(root.searchTimer); root.searchTimer = setTimeout(render, 200); } })) : '',
fpSel.size > 0 ? el('div', { className: 'mbgx-selbar' }, el('b', {}, `${fpSel.size} selected`),
list.some(x => !fpSel.has(fpKey(x))) && el('a', { href: 'javascript:;', onclick: () => { list.forEach(x => fpSel.add(fpKey(x))); render(); } }, `Select all ${list.length} matching`),
el('a', { href: 'javascript:;', onclick: () => { fpSel.clear(); render(); } }, 'Clear selection'),
el('button', { type: 'button', onclick: () => exportDialog('falsePositives', [{ label: 'Selected', list: allow.filter(x => fpSel.has(fpKey(x))) }]) }, 'Export…'),
el('button', { type: 'button', className: 'mbgx-fp-sel-remove', onclick: () => removeKeys(new Set(fpSel), `Remove ${fpSel.size} selected false positives?`) }, 'Remove')) : '',
table,
pages > 1 ? el('p', { className: 'mbgx-pager' },
el('button', { type: 'button', disabled: fpPage === 0, onclick: () => { fpPage--; render(); } }, '‹ Previous'), ` Page ${fpPage + 1} of ${pages} `,
el('button', { type: 'button', disabled: fpPage >= pages - 1, onclick: () => { fpPage++; render(); } }, 'Next ›')) : '',
el('p', {},
allow.length ? el('button', { type: 'button', onclick: () => exportDialog('falsePositives') }, 'Export false positives…') : '',
el('button', { type: 'button', onclick: () => chooseImportFile(render) }, 'Import false positives file…'),
allow.length ? el('button', { type: 'button', onclick: () => removeKeys(new Set(allow.map(fpKey)), `Clear all ${allow.length} false positives?`) }, 'Clear all false positives…') : ''));
typing();
renderTabs();
};
render();
return root;
};
// ------------------------------------------------------------ Diagnostics
BUILD.diagnostics = () => {
const env = environment(), live = el('pre', { className: 'mbgx-live' });
const avg = t => (t.n ? (t.total / t.n).toFixed(t.total / t.n < 1 ? 3 : 1) : '–');
const pp = t => { const p = percentiles(t); return p.p50 == null ? '–' : `${p.p50}/${p.p95}`; };
const speed = el('span', { className: 'mbgx-muted' });
const failed = el('fieldset', { className: 'mbgx-failed-list' }), errors = el('fieldset', { className: 'mbgx-errors-list' });
let failedSig = null, errorsSig = null;
const retry = st => { st.failures = 0; return ensureHash(st).catch(() => {}); };
update = () => {
live.textContent = `Page (${PAGE}): ${stats.imagePosts} image posts · ${stats.buttons} buttons · hashed ${stats.hashOK} · failed ${stats.hashErrors} · queued ${queue.length + active} · filtered ${document.querySelectorAll('.mbgx-filtered').length} · near misses ${stats.nearMisses}` +
(stats.firstBatch ? `\nHashing time: first batch ${batchText(stats.firstBatch)}` + (stats.lastBatch !== stats.firstBatch ? ` · latest ${batchText(stats.lastBatch)}` : '') : '') +
`\nPer thumbnail: fetch ${avg(stats.fetchMs)} ms (${stats.directFetches} direct${directFetch ? '' : ', switched to GM requests'}) · decode ${avg(stats.decodeMs)} ms · hash ${avg(stats.hashMs)} ms · match ${avg(stats.matchMs)} ms (max ${stats.matchMs.n ? stats.matchMs.max.toFixed(2) : '–'}) · extended ${avg(stats.extMs)} ms (max ${stats.extMs.n ? stats.extMs.max.toFixed(2) : '–'})` +
`\nTypical (median / 95%): fetch ${pp(stats.fetchMs)} · decode ${pp(stats.decodeMs)} (waiting ${pp(stats.bitmapMs)}, drawing ${pp(stats.drawMs)}) · extended ${pp(stats.extMs)} ms · ` +
`queued ${pp(stats.queueMs)} · found to decided ${pp(stats.decisionMs)} ms (in view ${pp(stats.visibleDecisionMs)}) · clock step ${timerStep()} ms` +
`\nPage busy during hashing: ${stats.stalls.n} delays over 50 ms (worst ${stats.stalls.n ? Math.round(stats.stalls.max) : 0} ms) · ` +
`${(stats.thumbBytes / 1024).toFixed(0)} KB of thumbnails · up to ${stats.peakDecodes} decodes at once · greyscale kept ${(memoryEstimate().measured.grayRetainedBytes / 1024).toFixed(0)} KB` +
`\nMatches by method: ${Object.entries(stats.byMethod).map(([k, n]) => `${METHOD[k]} ${n}`).join(' · ') || 'none yet'}` +
`\nEdited copies checked ${pool.state === 'on' ? (pool.workers.length === 1 ? 'in a background worker' : `in ${pool.workers.length} background workers`) + (pool.indexMs ? ` (prepared in ${pool.indexMs} ms)` : '')
+ (pool.jobs.size + pool.waiting.length ? ` · ${pool.jobs.size + pool.waiting.length} pictures being checked` : '')
: pool.state === 'starting' ? 'on the page while background workers start'
: pool.state === 'off' && !pool.reason ? 'in a background worker once there are references to check them against' : `on the page (background workers: ${pool.reason})`}` +
(pool.state !== 'on' && refDB.anyExt && !refDB.ov ? ' · crop index being prepared' : '') +
`\nReferences: ${refDB.seeds.length} by picture, ${refDB.md5Seeds.length} by MD5${cfg.md5Match ? '' : ' (off in general)'}, of ${seeds.length} (${cats.filter(c => c.enabled === false).length} disabled categories) · prepared in ${stats.prepMs} ms · fingerprints reused ${stats.cacheHits} · clones skipped ${stats.clonesSkipped} · script errors ${stats.scriptErrors}` +
(autoHash() ? '' : '\nBackground hashing is off (filtering disabled or no picture references); thumbnails are hashed only when you click "⊘ blacklist".');
const rows = [...posts.values()].filter(st => st.status === 'error' && st.root && st.root.isConnected);
const fs = rows.map(st => `${st.key}:${st.error}`).join();
if (fs !== failedSig) {
failedSig = fs;
failed.hidden = !rows.length;
failed.replaceChildren(el('legend', {}, `Failed thumbnails (${rows.length})`),
el('div', { className: 'mbgx-scroll' }, el('table', {},
el('tr', {}, ['Post', 'Error', ''].map(t => el('th', {}, t))),
rows.map(st => el('tr', { dataset: { mbgxKey: st.key } },
el('td', {}, goLink(st)),
el('td', { className: 'mbgx-error' }, st.error + (/HTTP 404/.test(st.error) ? ' (file deleted or thread pruned)' : '')),
el('td', {}, el('button', { type: 'button', className: 'mbgx-retry', onclick: e => { e.target.disabled = true; retry(st); } }, 'Retry')))))),
rows.length > 1 ? el('button', { type: 'button', onclick: e => { e.target.disabled = true; rows.forEach(retry); } }, 'Retry all') : '');
}
const es = recentErrors.map(x => `${x.where}:${x.error}:${x.count}`).join();
if (es !== errorsSig) {
errorsSig = es;
errors.hidden = !recentErrors.length;
errors.replaceChildren(el('legend', {}, `Script errors (${stats.scriptErrors})`),
el('div', { className: 'mbgx-scroll' }, el('table', {},
el('tr', {}, ['While', 'Error', 'Times'].map(t => el('th', {}, t))),
recentErrors.map(x => el('tr', {}, el('td', {}, ERROR_PLACES[x.where] || x.where), el('td', { className: 'mbgx-error' }, x.error), el('td', {}, String(x.count)))))),
el('p', { className: 'mbgx-muted' }, 'Full details, including where in the code, are in "Export diagnostics JSON".'));
}
};
return el('div', {},
el('p', { className: 'mbgx-muted' }, `Detected: ${env.xFamily ? env.xFamily.name + (env.xFamily.version ? ' ' + env.xFamily.version : '') : '4chan X family no'} · style script ${env.styleScript || 'none'} · ${env.manager} · canvas readback ${session.canvas.ok ? 'OK' : 'ALTERED/BLOCKED (' + (session.canvas.error || 'max deviation ' + session.canvas.maxDev) + ')'}`),
live, failed, errors,
el('fieldset', {}, el('legend', {}, 'Background worker'),
check('mbgx-workers', 'workers', 'Check edited copies in a background worker, where the browser allows it (keeps the page smooth)'), el('br'),
el('p', { className: 'mbgx-muted' }, 'Turn off only to find out whether they cause a problem.')),
el('fieldset', {}, el('legend', {}, 'Diagnostic data'),
check('mbgx-keeplog', 'keepLog', 'Keep a diagnostic log across pages (stored in this browser: up to 1,500 events and 30 page summaries)'), el('br'),
check('mbgx-verbose', 'verbose', 'Verbose log (one entry per hashed thumbnail)'), el('br'),
el('p', { className: 'mbgx-muted' }, 'Off (the default): only this page is logged, in memory, and nothing is stored. The box above always shows this page.'),
el('p', { className: 'mbgx-muted' }, 'Diagnostics help find out why something was or wasn\'t filtered, or why it was slow. ' +
'The anonymised export has only counts, timings, settings and versions: fit to share. The full export also contains what you did on 4chan, ' +
'for example the addresses of pages you opened (…/g/thread/123456789), post numbers, image hashes and MD5s of posts you saw, ' +
'your references with their posts, categories and tags, your false positives and your browser version. It is meant for troubleshooting on your own.'),
el('button', { type: 'button', onclick: () => download(`mbgx-diagnostics-anonymised-${today()}.json`, anonymousDiagnostics()) }, 'Export diagnostics (anonymised)'),
el('button', { type: 'button', onclick: () => ask('The full diagnostics export contains what you did on 4chan: addresses of pages you opened, post and thread numbers, ' +
'image hashes and MD5s of posts you saw, your references (with their posts, categories and tags), your false positives, your settings and your browser version.\n\n' +
'It is meant for troubleshooting on your own. To share diagnostics with someone, use the anonymised export instead.', 'Export full diagnostics')
.then(ok => ok && download(`mbgx-diagnostics-full-${Date.now()}.json`, diagnostics())) }, 'Export full diagnostics…'),
el('button', { type: 'button', title: "Re-time matching on this page's thumbnails now that it is quiet, to compare the script's own speed with the times measured while the page loaded",
onclick: e => { e.target.disabled = true; measureMatching(r => { speed.textContent = ` ${r}`; e.target.disabled = false; }); } }, 'Measure matching speed'), speed,
el('button', { type: 'button', onclick: () => ask('Clear the stored diagnostic log and page-session summaries?', 'Clear').then(ok => {
if (!ok) return;
pendingLogs = [];
write('logs', []);
write('sessions', []);
}) }, 'Clear diagnostics'),
el('span', { className: 'mbgx-muted' }, ` Stored log entries: ${read('logs', []).length}/${LOG_MAX}`)));
};
/** Before a list redraw: if its search box has focus, return a function that puts focus and cursor back. */
function keepTyping(root) {
const a = document.activeElement;
if (!a || !root.contains(a) || a.type !== 'search') return () => {};
const { className, selectionStart, selectionEnd } = a;
return () => {
const box = root.querySelector(`input[type=search].${className.split(' ')[0]}`);
if (box) { box.focus(); box.setSelectionRange(selectionStart, selectionEnd); }
};
}
/** A compact dropdown menu: [[label, action], …]; falsy entries are skipped. */
function menu(label, items, title = 'More actions') {
const m = el('details', { className: 'mbgx-menu' }, el('summary', { title }, label),
el('div', {}, items.filter(Boolean).map(([text, fn]) => el('button', { type: 'button', onclick: () => { m.open = false; guard('panel', fn); } }, text))));
return m;
}
document.body.append(backdrop, p);
document.addEventListener('keydown', onKey, true);
show(tab);
timer = setInterval(() => { if (update) update(); }, 1000);
// Redraw only if the archive list actually changed (a redraw would drop a half-typed search or a click).
refreshArchives(false).then(changed => { if (changed && tab === 'references' && document.getElementById('mbgx-panel')) show('references'); });
}
/**
* Whether a match shows a stub: its reference's category can choose, otherwise the general
* setting. Without one the post is hidden whole; copies in XT previews/inline quotes keep their stub.
*/
function stubsFor(m) {
return catSetting(m && m.seed && m.seed.cat, 'stubs');
}
/** After a change to the general setting or a category's: re-mark the posts already filtered. */
function applyStubMode() {
for (const st of posts.values()) {
const m = st.result && st.result.match;
if (m && st.target && st.target.classList.contains('mbgx-filtered')) st.target.classList.toggle('mbgx-nostub', !stubsFor(m));
}
}
/** Hidden-post count on every settings entry (there are two on vanilla pages: desktop and mobile). */
function updateControl() {
// Only what Better Filter alone hides: posts 4chan / XT hide anyway are not counted.
const n = [...document.querySelectorAll('.mbgx-filtered')].filter(e => !nativeHidden(e) && !e.closest(CLONE_CONTEXT)).length;
for (const c of document.querySelectorAll('.mbgx-count')) {
c.textContent = n ? String(n) : '';
c.title = n ? `${n} post(s) hidden by Better Filter on this page` : '';
}
}
const SVG_NS = 'http://www.w3.org/2000/svg';
function banIcon() {
const svg = document.createElementNS(SVG_NS, 'svg');
svg.setAttribute('class', 'icon mbgx-icon');
svg.setAttribute('viewBox', '0 0 512 512');
const circle = document.createElementNS(SVG_NS, 'circle');
const line = document.createElementNS(SVG_NS, 'line');
for (const [k, v] of Object.entries({ cx: 256, cy: 256, r: 212 })) circle.setAttribute(k, v);
for (const [k, v] of Object.entries({ x1: 106, y1: 106, x2: 406, y2: 406 })) line.setAttribute(k, v);
for (const e of [circle, line]) {
e.setAttribute('fill', 'none');
e.setAttribute('stroke', 'currentColor');
e.setAttribute('stroke-width', '56');
svg.append(e);
}
return svg;
}
/**
* Settings entry: an icon among 4chan X's header shortcuts (ordered by data-index, like XT's own
* Header.addShortcut), else "[Media filter]" in vanilla's top-right nav, else a small floating button.
* `floating` forces the button. Any earlier entry is removed first, so this can run again.
*/
let controlNodes = [];
function mountControl(floating = false) {
for (const n of controlNodes) n.remove();
controlNodes = [];
const open = () => guard('panel', panel);
const title = 'Better Filter: filter settings, references and diagnostics';
const count = () => el('span', { className: 'mbgx-count' });
const put = (...nodes) => { controlNodes.push(...nodes); return nodes; };
const xt = document.getElementById('shortcuts');
const settings = document.getElementById('settingsWindowLink');
const mobileSettings = document.getElementById('settingsWindowLinkMobile');
let kind = 'floating';
if (floating) {
// nothing else can be seen: the button below
} else if (xt) {
const INDEX = 880; // just before XT's menu button (900)
const link = el('a', { className: 'mbgx-control', href: 'javascript:;', title, onclick: open },
el('span', { className: 'icon--alt-text' }, 'Media filter'), banIcon(), count());
const [item] = put(el('span', { id: 'shortcut-mbgx', className: 'shortcut brackets-wrap', dataset: { index: INDEX } }, link));
const next = [...xt.querySelectorAll(':scope > [data-index]')].find(s => +s.dataset.index > INDEX);
xt.insertBefore(item, next || null);
kind = 'xt-header';
} else if (settings || mobileSettings) {
// Desktop: "[Settings] [Search] ..." -> "[Settings] [Media filter] [Search] ...". 4chan's mobile
// layout hides that bar and shows its own links instead: "Settings Filter Mobile Home".
if (settings) settings.after(...put(document.createTextNode('] ['), el('a', { className: 'mbgx-control', href: 'javascript:;', title, onclick: open }, 'Media filter', count())));
if (mobileSettings) mobileSettings.after(...put(document.createTextNode(' '), el('a', { className: 'mbgx-control mbgx-control-mobile', href: 'javascript:;', title, onclick: open }, 'Filter', count()), document.createTextNode(' ')));
kind = 'vanilla-nav';
}
if (kind === 'floating') document.body.append(...put(el('button', { type: 'button', className: 'mbgx-control mbgx-floating', title, onclick: open }, '⊘ Media filter ', count())));
updateControl();
return kind;
}
/**
* 4chan X builds its header when it is ready, which can be after Better Filter has started (with a
* userscript manager whose storage answers slowly, such as Greasemonkey 4), and it hides vanilla's
* nav. So the entry moves into its header whenever that appears without it; and once 4chan X has
* finished starting (or after a few seconds), an entry that can't be seen anywhere becomes the
* floating button.
*/
function checkControl(fallback = false) {
const sc = document.getElementById('shortcuts');
if (sc && !sc.contains(document.getElementById('shortcut-mbgx'))) session.control = mountControl();
else if (fallback && session.control !== 'floating' && !controlNodes.some(n => n.nodeType === 1 && n.isConnected && n.getClientRects().length)) {
session.control = mountControl(true);
}
}
// ---------------------------------------------------------------- startup
// The hide rule is deliberately over-specific: OneeChan forces replies visible with
// ":root.reply-fit-width .reply.post{display:block!important}" (specificity 0,4,0).
// OneeChan also hides every new checkbox and inserts a styled div.riceCheck after it; the panel
// keeps its native checkboxes and hides those copies, so each option shows one box.
/**
* Window colours. "Match the page" (the default) takes them from the page itself: a post's
* background, the text and the link colours. So the windows follow 4chan's board styles, 4chan
* X/XT's themes, StyleChan and OneeChan without knowing any of them, and change with
* them (they are read again whenever a window opens). Dark and Light are built in. Everything else
* (lines, fields, tags, the selection bar) is mixed from these four colours.
*/
const THEMES = {
dark: { bg: [36, 36, 36], fg: [238, 238, 238], link: [153, 204, 255] },
light: { bg: [246, 246, 246], fg: [29, 29, 29], link: [31, 95, 191] },
};
/**
* A computed colour -> [r, g, b, alpha], or null: 'rgb(1, 2, 3)', 'rgba(1, 2, 3, 0.5)', 'rgb(1 2 3 / 50%)',
* and 'color(srgb 0.1 0.2 0.3 / 0.5)' (how newer browsers report some colours). Anything else: null.
*/
function parseColour(text) {
const alpha = (v, pct) => (v === undefined ? 1 : Number(v) / (pct ? 100 : 1));
let m = /^rgba?\(\s*([\d.]+)[\s,]+([\d.]+)[\s,]+([\d.]+)(?:\s*[,/]\s*([\d.]+)(%?))?\s*\)$/.exec(text || '');
if (m) return [Number(m[1]), Number(m[2]), Number(m[3]), alpha(m[4], m[5])];
m = /^color\(srgb\s+([\d.]+)\s+([\d.]+)\s+([\d.]+)(?:\s*\/\s*([\d.]+)(%?))?\s*\)$/.exec(text || '');
if (m) return [m[1], m[2], m[3]].map(v => Math.round(Math.min(1, Number(v)) * 255)).concat(alpha(m[4], m[5]));
return null;
}
const mix = (x, y, t) => x.map((v, i) => Math.round(v + (y[i] - v) * t));
const luminance = c => {
const [r, g, b] = c.map(v => { v /= 255; return v <= 0.03928 ? v / 12.92 : ((v + 0.055) / 1.055) ** 2.4; });
return 0.2126 * r + 0.7152 * g + 0.0722 * b;
};
const contrast = (x, y) => { const p = luminance(x), q = luminance(y); return (Math.max(p, q) + 0.05) / (Math.min(p, q) + 0.05); };
const css = c => `rgb(${c[0]},${c[1]},${c[2]})`;
/** The page's own colours: a post's background (see-through layers mixed down), its text and a link. */
function pageColours() {
const sample = document.querySelector('.post.reply') || document.querySelector('.reply') || document.body;
if (!sample) return null;
const layers = [];
for (let e = sample; e; e = e.parentElement) {
const c = parseColour(getComputedStyle(e).backgroundColor);
if (c && c[3] > 0) { layers.push(c); if (c[3] >= 1) break; }
}
let bg = [255, 255, 255];
for (const c of layers.reverse()) bg = mix(bg, c, c[3]);
const text = parseColour(getComputedStyle(sample.querySelector('.postMessage') || sample).color);
const a = document.querySelector('.postMessage a.quotelink, #boardNavDesktop a, .boardList a, a.backlink, .navLinks a, a[href]');
const link = a && parseColour(getComputedStyle(a).color);
if (!text) return null;
let fg = mix(bg, text, text[3]);
// too faint to read comfortably (under 4.5:1): plain light or dark text, whichever reads better
if (contrast(fg, bg) < 4.5) fg = contrast(THEMES.dark.fg, bg) > contrast(THEMES.light.fg, bg) ? THEMES.dark.fg : THEMES.light.fg;
let ln = link ? mix(bg, link, link[3]) : fg;
if (contrast(ln, bg) < 2.5) ln = fg;
return { bg, fg, link: ln };
}
/**
* Write the colours of the chosen theme into a style element (CSS variables used by every window).
* If the page's colours can't be read, the built-in Dark ones are used; if even this fails, the
* defaults in CSS (Dark) stay in place, so the windows always have colours.
*/
function applyTheme() {
let page = null;
try { page = cfg.theme !== 'dark' && cfg.theme !== 'light' && pageColours(); } catch (e) { page = null; } // e.g. a style script mid-change
const t = page || THEMES[cfg.theme === 'light' ? 'light' : 'dark'];
const dark = luminance(t.bg) < 0.4, edge = dark ? [0, 0, 0] : [255, 255, 255];
// lines mixed from the text colour, but always visible (low-contrast themes need a stronger mix)
const line = (t0, min) => { let k = t0; while (k < 0.9 && contrast(mix(t.bg, t.fg, k), t.bg) < min) k += 0.05; return mix(t.bg, t.fg, k); };
const v = {
bg: t.bg, fg: t.fg, link: t.link,
field: mix(t.bg, edge, dark ? 0.4 : 0.7), // inputs, code boxes
menu: mix(t.bg, edge, 0.25), // drop-down menus, the phone tooltip
raised: mix(t.bg, t.fg, 0.1), // the open tab, tags
line: line(0.18, 1.3), // table rows
border: line(0.38, 1.9), // fieldsets, tabs, fields
frame: line(0.55, 2.6), // the window's own border
muted: mix(t.fg, t.bg, 0.32),
select: mix(t.bg, t.link, 0.22), // the selection bar
ok: mix(t.bg, dark ? [80, 200, 130] : [40, 150, 90], 0.25), // the "pieces" tag
warn: dark ? [255, 204, 136] : [150, 80, 0],
error: dark ? [255, 153, 153] : [190, 20, 20],
};
let style = document.getElementById('mbgx-theme');
if (!style) (document.head || document.documentElement).append(style = el('style', { id: 'mbgx-theme' }));
style.textContent = `:root:root{${Object.entries(v).map(([k, c]) => `--mbgx-${k}:${css(c)}`).join(';')}}` + // above the defaults in CSS
`.mbgx-panel,.mbgx-touch-tip{color-scheme:${dark ? 'dark' : 'light'}}`;
}
const CSS = `
:root{--mbgx-bg:#242424;--mbgx-fg:#eee;--mbgx-link:#9cf;--mbgx-field:#161616;--mbgx-menu:#1b1b1b;--mbgx-raised:#3a3a3a;--mbgx-line:#4d4d4d;--mbgx-border:#6f6f6f;--mbgx-frame:#909090;--mbgx-muted:#aaa;--mbgx-select:#2d3a52;--mbgx-ok:#2d4a3a;--mbgx-warn:#fc8;--mbgx-error:#f99}
.mbgx-action{cursor:pointer;white-space:nowrap;text-decoration:none;opacity:.35;transition:opacity .15s}
.post:hover .mbgx-action{opacity:.8}
.mbgx-action:hover,.mbgx-action:focus{opacity:1;text-decoration:underline}
.mbgx-control .mbgx-icon{vertical-align:middle}
#shortcut-mbgx .mbgx-icon{width:15px;height:15px;vertical-align:-3px}
:root.shortcut-icons #shortcut-mbgx .icon--alt-text,:root:not(.shortcut-icons) #shortcut-mbgx .mbgx-icon{display:none}
.mbgx-count:not(:empty){margin-left:2px;font-size:10px;font-weight:bold}
.mbgx-control:not(.mbgx-floating) .mbgx-count:not(:empty)::before{content:'('}
.mbgx-control:not(.mbgx-floating) .mbgx-count:not(:empty)::after{content:')'}
#shortcut-mbgx .mbgx-count:not(:empty)::before,#shortcut-mbgx .mbgx-count:not(:empty)::after{content:none}
:root .postContainer.mbgx-filtered.mbgx-filtered:not(.post-hidden):not(#qp *):not(.inline *):not(#quote-preview *)>.post{display:none!important}
#qp .mbgx-stub .mbgx-cmd,.inline .mbgx-stub .mbgx-cmd{display:none!important}
.mbgx-quote-filtered{text-decoration:underline line-through!important}
:root.hide-backlinks a.backlink.mbgx-quote-filtered,:root.hide-backlinks a.backlink.mbgx-quote-filtered+.hashlink{display:none!important}
.mbgx-tag-list label{display:inline-block;margin:2px 10px 2px 0}
:root .thread.mbgx-filtered.mbgx-filtered>:not(.mbgx-stub):not(.stub){display:none!important}
.thread[hidden]>.mbgx-stub{display:none!important}
.thread:has(>.stub)>.mbgx-stub{display:none!important}
#threads>.thread.mbgx-filtered>.mbgx-stub{display:block;padding:6px 4px;text-align:center}
#threads>.thread.mbgx-filtered>.mbgx-stub .mbgx-cmd{margin:0 3px}
.mbgx-stub{display:table;margin:1px 0;padding:2px 6px;font-size:11px;line-height:1.6}
.mbgx-stub-label{opacity:.6}
.mbgx-cmd{cursor:pointer;text-decoration:none;margin-left:5px;padding:0 4px;border-radius:2px;white-space:nowrap;opacity:.6;transition:opacity .12s,background-color .12s}
.mbgx-cmd:hover,.mbgx-cmd:focus{opacity:1;text-decoration:underline;background-color:rgba(128,128,128,.25);outline:none}
.post-hidden>.mbgx-stub{display:none!important}
:root .postContainer.mbgx-filtered.mbgx-nostub:not(.post-hidden):not(#qp *):not(.inline *),:root .thread.mbgx-filtered.mbgx-nostub{display:none!important}
#qp .mbgx-action,.inline .mbgx-action,.inlined .mbgx-action,#quote-preview .mbgx-action{display:none!important}
.mbgx-floating{position:fixed;right:12px;bottom:36px;z-index:99998;padding:3px 7px;cursor:pointer;font:11px sans-serif;opacity:.6}
.mbgx-floating:hover{opacity:1}
.mbgx-lookup{position:fixed;left:12px;bottom:12px;top:auto;transform:none;width:auto;max-width:min(90vw,520px);z-index:99998;padding:6px 10px;font:12px sans-serif}
.mbgx-panel{position:fixed;z-index:999999;top:4vh;left:50%;transform:translateX(-50%);width:min(94vw,max(760px,min(1280px,72vw)));max-height:92vh;overflow:auto;box-sizing:border-box;padding:14px 16px;background:var(--mbgx-bg);color:var(--mbgx-fg);border:2px solid var(--mbgx-frame);box-shadow:0 4px 18px #0006;font:13px/1.45 sans-serif;text-align:left}
.mbgx-backdrop{position:fixed;inset:0;z-index:999998;background:#0009}
.mbgx-backdrop.mbgx-export-backdrop{z-index:1000000;background:#0006}
.mbgx-backdrop.mbgx-dialog-backdrop{z-index:1000002;background:#0006}
.mbgx-panel.mbgx-dialog{z-index:1000003;top:22%;width:min(440px,92vw)}
.mbgx-dialog p{white-space:pre-line}
.mbgx-panel.mbgx-export{z-index:1000001;top:16%;width:min(500px,92vw);max-height:80vh}
.mbgx-export .mbgx-live{max-height:30vh;overflow:auto}
.mbgx-exp-row{margin:8px 0}
.mbgx-exp-row .mbgx-muted{margin-left:22px;font-size:12px}
.mbgx-exp-row.mbgx-disabled{opacity:.45}
.mbgx-panel input.mbgx-scope{width:84px}
.mbgx-panel h3{margin:0 0 6px;font-size:16px}
.mbgx-panel a{color:var(--mbgx-link)}
.mbgx-panel fieldset{min-width:0;border:1px solid var(--mbgx-border)!important;margin:8px 0;padding:6px 10px}
.mbgx-panel button{cursor:pointer;margin:3px 4px 3px 0;padding:2px 8px;font:inherit;background:var(--mbgx-raised);color:var(--mbgx-fg);border:1px solid var(--mbgx-border);border-radius:3px}
.mbgx-panel button:hover,.mbgx-panel button:focus-visible{border-color:var(--mbgx-link)}
.mbgx-panel button:disabled{opacity:.5;cursor:default}
.mbgx-panel input[type=number]{width:52px}
.mbgx-panel .riceCheck{display:none!important}
.mbgx-panel input[type=checkbox]{display:inline-block!important;width:auto!important;height:auto!important;-moz-appearance:auto;-webkit-appearance:auto;appearance:auto}
.mbgx-panel.mbgx-panel select,.mbgx-panel.mbgx-panel input,.mbgx-panel.mbgx-panel textarea{transition:none!important;background:var(--mbgx-field)!important;color:var(--mbgx-fg)!important;border:1px solid var(--mbgx-border)!important;font:inherit!important}
.mbgx-panel select{appearance:auto} /* StyleChan: select{appearance:none} */
.mbgx-panel textarea{width:100%;height:140px;box-sizing:border-box;font:12px monospace}
.mbgx-panel table{border-collapse:collapse;width:100%}
@media (max-width:600px){.mbgx-panel table{display:block;max-width:100%;overflow-x:auto}} /* phones: wide tables scroll inside, not the window */
.mbgx-panel th,.mbgx-panel td{border-bottom:1px solid var(--mbgx-line);padding:2px 5px;text-align:left}
.mbgx-panel code,.mbgx-live{font:12px monospace}
.mbgx-live{white-space:pre-wrap;overflow-wrap:anywhere;background:var(--mbgx-field);padding:6px;margin:6px 0}
.mbgx-muted{color:var(--mbgx-muted)}
.mbgx-add-warn{color:var(--mbgx-warn);margin:4px 0}
.mbgx-error{color:var(--mbgx-error)}
.mbgx-tabs{display:flex;flex-wrap:wrap;gap:2px;border-bottom:1px solid var(--mbgx-border);margin:4px 0 8px}
.mbgx-panel .mbgx-tab{margin:0 0 -1px;padding:4px 10px;border:1px solid transparent;border-bottom:none;border-radius:3px 3px 0 0;background:none;color:inherit;cursor:pointer}
.mbgx-panel .mbgx-tab[aria-selected=true]{border-color:var(--mbgx-border);background:var(--mbgx-raised);font-weight:bold}
.mbgx-toolbar{display:flex;flex-wrap:wrap;gap:4px;align-items:center;margin:2px 0 6px}
.mbgx-toolbar .mbgx-search{flex:1 1 160px;min-width:120px}
.mbgx-selbar{display:flex;flex-wrap:wrap;gap:6px;align-items:center;margin:4px 0;padding:4px 6px;border-radius:3px;background:var(--mbgx-select)}
.mbgx-catbar{margin:2px 0 6px;padding:3px 6px;border-left:3px solid var(--mbgx-link)}
.mbgx-catbar p{margin:0 0 2px}
.mbgx-catbar-actions{display:flex;flex-wrap:wrap;gap:2px 4px;align-items:center}
.mbgx-cat-hint{margin:2px 0 6px}
.mbgx-menu{position:relative;display:inline-block}
.mbgx-menu>summary{list-style:none;cursor:pointer;padding:1px 9px;border:1px solid var(--mbgx-border);border-radius:3px;background:var(--mbgx-raised);white-space:nowrap}
.mbgx-menu>summary::-webkit-details-marker{display:none}
.mbgx-menu>div{position:absolute;right:0;z-index:3;display:flex;flex-direction:column;min-width:210px;padding:4px;background:var(--mbgx-menu);border:1px solid var(--mbgx-border);box-shadow:0 3px 10px #0008}
.mbgx-panel .mbgx-menu>div button{margin:1px 0;text-align:left}
.mbgx-tag{display:inline-block;margin:1px 2px 1px 0;padding:0 6px;border-radius:8px;background:var(--mbgx-raised);font-size:11px}
.mbgx-tag.mbgx-has-pieces{background:var(--mbgx-ok)}
.mbgx-panel button.mbgx-tag{margin:1px 2px 1px 0;padding:0 6px;border:1px solid transparent;border-radius:8px;background:var(--mbgx-raised);color:var(--mbgx-fg);font-size:11px;line-height:inherit;max-width:14em;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;vertical-align:middle}
.mbgx-panel td.mbgx-tags-cell{max-width:22em;min-width:5em}
.mbgx-panel button.mbgx-tag:hover,.mbgx-panel button.mbgx-tag:focus-visible{border-color:var(--mbgx-link)}
.mbgx-panel button.mbgx-tag-plus,.mbgx-panel button.mbgx-tag-more{background:none;border-color:var(--mbgx-line);color:var(--mbgx-muted)}
.mbgx-tag-row{display:flex;gap:8px;align-items:center;margin:4px 0}
.mbgx-tag-row .mbgx-tag-name{flex:1 1 auto;min-width:0}
.mbgx-form .mbgx-tag-row label{display:inline-flex;align-items:center;gap:4px;margin:0;white-space:nowrap}
.mbgx-tag-editor label{margin:6px 0}
.mbgx-tag-editor .mbgx-tag-rows{max-height:45vh;overflow:auto;padding-right:4px}
.mbgx-tag.mbgx-no-pieces{background:none;border:1px solid var(--mbgx-line);color:var(--mbgx-muted)}
.mbgx-subopt{display:inline-block;margin-left:22px}
.mbgx-panel.mbgx-dialog{max-height:74vh}
.mbgx-panel.mbgx-dialog:has(.mbgx-cat-settings),.mbgx-panel.mbgx-dialog:has(.mbgx-tag-editor){top:4vh;max-height:92vh}
.mbgx-panel.mbgx-dialog:has(.mbgx-cat-settings){width:min(640px,94vw)}
.mbgx-cat-settings select{max-width:100%}
.mbgx-form.mbgx-cat-settings fieldset>label{display:flex;flex-wrap:wrap;gap:4px 8px;align-items:center;justify-content:space-between;margin:4px 0}
.mbgx-touch-tip{position:fixed;z-index:1000004;max-width:min(360px,90vw);padding:6px 8px;background:var(--mbgx-menu);color:var(--mbgx-fg);border:1px solid var(--mbgx-border);box-shadow:0 3px 10px #0008;font:12px/1.4 sans-serif;white-space:pre-wrap;text-align:left}
.mbgx-panel tr.mbgx-disabled,.mbgx-catbar.mbgx-disabled{opacity:.5}
.mbgx-pager{text-align:center}
.mbgx-form label{display:block;margin:5px 0}
.mbgx-form input[type=text],.mbgx-form select{max-width:100%;box-sizing:border-box}
.mbgx-form input[type=text]{width:100%}
.mbgx-dialog .mbgx-form{margin:6px 0}
.mbgx-rules label{white-space:nowrap}
.mbgx-panel fieldset table{margin-top:2px}
.mbgx-scroll{max-height:max(220px,45vh);overflow:auto}
.mbgx-scroll th{position:sticky;top:0;background:var(--mbgx-bg);z-index:1}
.postContainer.mbgx-flash{outline:2px solid #e8a33d;outline-offset:2px;transition:outline-color .3s}
.mbgx-form label.mbgx-disabled{opacity:.45}
.mbgx-panel textarea.mbgx-xf-text{height:110px}
:root.mbgx-xbusy #overlay,:root.mbgx-xbusy #xt-settings-overlay{visibility:hidden!important}
`;
function start() {
if (typeof GM_addStyle === 'function') GM_addStyle(CSS);
else document.head.append(el('style', { textContent: CSS }));
guard('control', applyTheme);
prepareData();
session.canvas = canvasSelfTest();
session.environment = environment();
watchBrowserStats();
applyStubMode();
session.control = guard('control', mountControl);
document.addEventListener('4chanXInitFinished', () => guard('control', () => checkControl(true)));
for (const ms of [5000, 15000]) setTimeout(() => guard('control', () => checkControl(true)), ms);
guard('control', touchTips);
const scanStart = performance.now();
guard('scan', scan);
stats.firstScanMs = round(performance.now() - scanStart); // every post on the page, once (diagnostics)
log('page-start', { version: VERSION, url: location.href, environment: session.environment, canvas: session.canvas,
posts: stats.posts, imagePosts: stats.imagePosts, noImage: stats.noImage, clonesSkipped: stats.clonesSkipped, buttons: stats.buttons,
references: activeSeeds().length, autoHash: autoHash() });
if (!session.canvas.ok) log('canvas-warning', session.canvas);
// New posts and tiles (thread updater, XT inline/expand, index pages, catalog re-sorting): only the
// added nodes are scanned, batched every 100 ms. Our own nodes are ignored.
const added = new Set();
let scheduled = false;
new MutationObserver(records => {
if (!document.getElementById('shortcut-mbgx') && document.getElementById('shortcuts')) guard('control', checkControl);
for (const r of records) {
// "⊘ blacklist" stays last on the file line: 4chan X/XT append their image-search ("sauce")
// links there whenever they get to a post, which may be after us.
if (r.addedNodes.length && r.target.nodeType === 1 && r.target.classList.contains('fileText')) keepButtonLast(r.target);
for (const n of r.addedNodes) {
if (n.nodeType === 1 && !n.matches('.mbgx-stub, .mbgx-action, .mbgx-panel, .mbgx-backdrop, .mbgx-dialog')) added.add(n);
}
}
if (!added.size || scheduled) return;
scheduled = true;
setTimeout(() => {
scheduled = false;
const nodes = [...added];
added.clear();
for (const n of nodes) if (n.isConnected) guard('scan', () => scan(n));
scheduleQuotes();
}, 100);
}).observe(document.body, { childList: true, subtree: true });
addEventListener('pagehide', flush);
// Re-prioritise the hashing queue around wherever you scroll to (sorted lazily on the next request).
addEventListener('scroll', () => { if (queue.length) queueSorted = false; }, { passive: true });
document.addEventListener('visibilitychange', () => { if (document.visibilityState === 'hidden') flush(); });
}
guard('start', start);
})();