Automated background scanner for capturing ATW user posts that match someone's interests (based on keyword matching) from the site's activity feed. Provides an easy to use floating UI dashboard that allows 3-state multi-tag filtering, persistent storage, and JSON export/import.
// ==UserScript==
// @name ATW Custom Activity Feed
// @namespace http://tampermonkey.net/
// @version 1.2
// @description Automated background scanner for capturing ATW user posts that match someone's interests (based on keyword matching) from the site's activity feed. Provides an easy to use floating UI dashboard that allows 3-state multi-tag filtering, persistent storage, and JSON export/import.
// @author echo17
// @match https://www.allthingsworn.com/*
// @grant GM_setValue
// @grant GM_getValue
// @grant GM_addValueChangeListener
// @license MIT
// ==/UserScript==
(function() {
'use strict';
// ==========================================
// SYSTEM CONFIGURATION (Unchangeable Defaults)
// ==========================================
const SYS_CONFIG = {
mediaBaseUrl: 'https://www.allthingsassets.com/img/',
storageKey: 'atw-af-storage-v1',
settingsKey: 'atw-af-config-v1',
useLZWCompression: false,
debug: false
};
function debugLog(...args) { if (SYS_CONFIG.debug) console.log(...args); }
const myTabId = Date.now().toString(36) + Math.random().toString(36).substr(2);
// ==========================================
// DEFAULT CONFIGURATION
// ==========================================
const DEFAULT_CONFIG = {
// 1. CAPTURE RULES: A post MUST contain at least one of these to be saved.
captureKeywords: ["thong","juicy", "creamy", "🍋", "drive"],
// 2. EXCLUDE RULES: A post MUST NOT contain any of these.
excludeKeywords: ["Something I don't want to see", "Something else I don't want to see"],
// 3. TAGGING RULES: Applied ONLY to posts that passed the Capture/Exclude rules above.
// These words will NOT trigger a save on their own!
tags: {
"Instant Content": ["Created Instant Content", "Reposted Instant Conten"],
"Listing": ["Created a listing", "Reposted a listing"],
"Photo": ["Added a new photo"]
},
maxPages: 1000,
delayMs: 1000,
cycleDelayMinutes: 10,
baseUrl: '/activities?page=1&type=country&country=UK&sort=latest',
};
// ==========================================
// UNIFIED GLOBAL STATE
// ==========================================
let state = {
config: JSON.parse(JSON.stringify(DEFAULT_CONFIG)),
dbCorrupted: false,
isRunning: false, cycleTimer: null, currentPage: 1, currentUrl: DEFAULT_CONFIG.baseUrl,
blacklistedSignatures: new Set(), savedResults: [], users: {}, nextUserId: 1,
masterTabId: null, lastStatusText: "Idle", isHistoricalScrape: false, timeLimitMs: 0,
historicalStartStr: null, historicalDays: 7, lastCycleTimestamp: 0, currentCycleNewestTimestamp: 0
};
let renderState = { filteredResults: [], currentlyRendered: 0, chunkSize: 100, customFilterText: '', sortOrder: 'desc', tagStates: {} };
// ==========================================
// ASYNC YIELDING HELPER
// ==========================================
const yieldThread = () => new Promise(r => setTimeout(r, 1));
// ==========================================
// HELPERS & HASHING
// ==========================================
function getLocalIsoString(date) {
const pad = n => n.toString().padStart(2, '0');
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}T${pad(date.getHours())}:${pad(date.getMinutes())}`;
}
function hashString(str) {
let hash = 0;
for (let i = 0, len = str.length; i < len; i++) { hash = (hash << 5) - hash + str.charCodeAt(i); hash |= 0; }
return hash.toString(36);
}
function getMatchedTags(normalizedText) {
let matches = [];
for (const [tagName, keywords] of Object.entries(state.config.tags)) {
if (keywords.some(kw => normalizedText.includes(kw.toLowerCase()))) matches.push(tagName);
}
return matches;
}
function encodeCursor(dateObj) {
const pad = (n) => n.toString().padStart(2, '0');
const formatted = `${dateObj.getFullYear()}-${pad(dateObj.getMonth()+1)}-${pad(dateObj.getDate())} ${pad(dateObj.getHours())}:${pad(dateObj.getMinutes())}:${pad(dateObj.getSeconds())}`;
return btoa(`${formatted}|99999999`);
}
function decodeCursorDate(base64Str) {
try { return new Date(atob(base64Str).split('|')[0].replace(' ', 'T')); } catch(e) { return null; }
}
function getRelativeMediaPath(url) {
if (!url) return ""; const match = url.match(/\/img\/(.*)/); return match ? match[1] : url.split('/').pop();
}
function compressHtmlPayload(htmlString) {
if (!htmlString) return ""; let minified = htmlString;
minified = minified.replace(/https:\/\/(www\.)?(test|test2|testassets)\.com/gi, '');
minified = minified.replace(/<span class="activity-description[^>]*>([\s\S]*?)<\/span>/gi, "$1");
minified = minified.replace(/<a href="\/listing\/[^>]*>([\s\S]*?)<\/a>/gi, "$1");
minified = minified.replace(/<a[^>]*>a new photo<\/a>/gi, "a new photo");
minified = minified.replace(/\n\s*>\s*/g, ' > '); minified = minified.replace(/\s{2,}/g, ' ');
return minified.trim();
}
function lzwDecompressLegacy(str) {
if (!str) return str;
let dict = new Map(), data = str.split(""), currChar = data[0], oldPhrase = currChar, out = [currChar], code = 256, phrase;
for (let i = 1; i < data.length; i++) {
let currCode = data[i].charCodeAt(0);
if (currCode < 256) { phrase = data[i]; } else { phrase = dict.has(currCode) ? dict.get(currCode) : (oldPhrase + currChar); }
out.push(phrase); currChar = phrase.charAt(0);
if (code < 65536) { dict.set(code, oldPhrase + currChar); code++; } oldPhrase = phrase;
}
return out.join("");
}
// ==========================================
// UNIFIED STORAGE MANAGEMENT
// ==========================================
async function saveStateAsync() {
if (state.dbCorrupted) return;
await yieldThread();
const stateToSave = {
config: state.config,
currentPage: state.currentPage, currentUrl: state.currentUrl,
blacklistedSignatures: Array.from(state.blacklistedSignatures),
savedResults: state.savedResults, users: state.users, nextUserId: state.nextUserId,
masterTabId: state.masterTabId, lastStatusText: state.lastStatusText,
isHistoricalScrape: state.isHistoricalScrape, timeLimitMs: state.timeLimitMs,
historicalStartStr: state.historicalStartStr, historicalDays: state.historicalDays,
lastCycleTimestamp: state.lastCycleTimestamp
};
let jsonPayload = JSON.stringify(stateToSave);
if (SYS_CONFIG.useLZWCompression && typeof LZString !== 'undefined') {
jsonPayload = "LZS|" + LZString.compressToUTF16(jsonPayload);
}
GM_setValue(SYS_CONFIG.storageKey, jsonPayload);
}
function loadState() {
let saved = GM_getValue(SYS_CONFIG.storageKey);
if (saved) {
try {
if (saved.startsWith("LZS|") && typeof LZString !== 'undefined') {
saved = LZString.decompressFromUTF16(saved.substring(4));
} else if (saved.startsWith("LZW2|") || saved.startsWith("LZW|")) {
saved = lzwDecompressLegacy(saved.substring(saved.indexOf('|') + 1));
}
const parsed = JSON.parse(saved);
if (parsed.config) {
state.config.captureKeywords = Array.isArray(parsed.config.captureKeywords) ? parsed.config.captureKeywords : state.config.captureKeywords;
state.config.excludeKeywords = Array.isArray(parsed.config.excludeKeywords) ? parsed.config.excludeKeywords : state.config.excludeKeywords;
state.config.tags = (parsed.config.tags && typeof parsed.config.tags === 'object' && !Array.isArray(parsed.config.tags)) ? parsed.config.tags : state.config.tags;
state.config.maxPages = Number.isInteger(parsed.config.maxPages) ? parsed.config.maxPages : state.config.maxPages;
state.config.delayMs = Number.isInteger(parsed.config.delayMs) ? parsed.config.delayMs : state.config.delayMs;
state.config.cycleDelayMinutes = Number.isInteger(parsed.config.cycleDelayMinutes) ? parsed.config.cycleDelayMinutes : state.config.cycleDelayMinutes;
state.config.baseUrl = typeof parsed.config.baseUrl === 'string' ? parsed.config.baseUrl : state.config.baseUrl;
}
state.currentPage = parsed.currentPage || 1; state.currentUrl = parsed.currentUrl || state.config.baseUrl;
state.blacklistedSignatures = new Set(parsed.blacklistedSignatures || []);
state.savedResults = parsed.savedResults || []; state.users = parsed.users || {};
state.nextUserId = parsed.nextUserId || 1; state.masterTabId = parsed.masterTabId || null;
state.lastStatusText = parsed.lastStatusText || 'Idle'; state.isHistoricalScrape = parsed.isHistoricalScrape || false;
state.timeLimitMs = parsed.timeLimitMs || 0; state.historicalStartStr = parsed.historicalStartStr || null;
state.historicalDays = parsed.historicalDays || 7; state.lastCycleTimestamp = parsed.lastCycleTimestamp || 0;
state.isRunning = parsed.masterTabId ? true : false;
state.dbCorrupted = false;
} catch (e) {
console.error("CRITICAL ERROR: Failed to load.", e);
state.dbCorrupted = true; state.isRunning = false;
const header = document.getElementById('scanner-header');
if (header) header.style.background = '#c0392b';
updateStatus("⚠️ DATABASE CORRUPTED. Reloading blocked to prevent wipe.");
}
}
}
// ==========================================
// UI CONSTRUCTION
// ==========================================
const ui = document.createElement('div');
ui.id = 'scanner-monitor-ui';
ui.innerHTML = `
<style>
#scanner-monitor-ui { position: fixed; top: 20px; right: 20px; width: 450px; height: 85vh; min-width: 320px; min-height: 250px; background: #fff; z-index: 999999; border-radius: 8px; box-shadow: 0 10px 30px rgba(0,0,0,0.3); display: flex; flex-direction: column; font-family: Arial, sans-serif; border: 1px solid #ccc; resize: both; overflow: hidden; transition: background 0.3s; }
#scanner-monitor-ui.maximized { top: 0 !important; left: 0 !important; right: 0 !important; bottom: 0 !important; width: 100vw !important; height: 100vh !important; border-radius: 0 !important; resize: none !important; }
#scanner-header { background: #2c3e50; color: #fff; padding: 12px 15px; margin: 0; font-size: 16px; font-weight: bold; display: flex; justify-content: space-between; cursor: move; user-select: none; }
#scanner-controls { background: #ecf0f1; padding: 10px; display: flex; gap: 6px; flex-wrap: wrap; border-bottom: 1px solid #bdc3c7; }
#scanner-controls button.main-btn { padding: 5px 10px; border: none; border-radius: 4px; cursor: pointer; background: #3498db; color: white; font-size: 12px; font-weight: bold; flex-grow: 1; transition: 0.1s; }
#scanner-controls button.main-btn:hover { background: #2980b9; }
#btn-toggle.takeover-mode { background: #f39c12 !important; } #btn-toggle.takeover-mode:hover { background: #d35400 !important; }
#btn-clear { background: #e74c3c !important; } #btn-clear:hover { background: #c0392b !important; }
#btn-export { background: #27ae60 !important; } #btn-export:hover { background: #2ecc71 !important; }
#btn-import { background: #8e44ad !important; } #btn-import:hover { background: #9b59b6 !important; }
#unified-time-panel { width: 100%; margin-top: 4px; display: flex; align-items: center; gap: 6px; background: #e2e8f0; padding: 6px; border-radius: 4px; border: 1px solid #cbd5e1; box-sizing: border-box; }
#unified-time-panel label { font-size: 11px; font-weight: bold; color: #334155; }
.tt-input { border: 1px solid #ccc; border-radius: 3px; font-size: 11px; padding: 2px; }
#scanner-filters-container { padding: 10px; background: #fff; border-bottom: 1px solid #eee; display: flex; flex-direction: column; gap: 8px; }
#scanner-tags-wrapper { display: flex; justify-content: space-between; align-items: flex-start; gap: 10px; }
#scanner-tags { display: flex; flex-wrap: wrap; gap: 6px; flex-grow: 1; }
.filter-tag { padding: 3px 8px; border-radius: 12px; font-size: 11px; font-weight: bold; cursor: pointer; transition: 0.2s; border: 1px solid transparent; user-select: none; }
.filter-tag:hover { filter: brightness(0.9); }
.filter-tag.active { background: #e67e22 !important; color: white !important; border-color: #d35400; }
.filter-tag.neutral { background: #e2e8f0 !important; color: #475569 !important; border-color: #cbd5e1; }
.filter-tag.excluded { background: #e74c3c !important; color: white !important; border-color: #c0392b; }
#btn-toggle-all-tags { background: #95a5a6; color: white; border: none; border-radius: 4px; padding: 3px 8px; font-size: 10px; font-weight: bold; cursor: pointer; transition: 0.2s; white-space: nowrap; }
.filter-row { display: flex; gap: 8px; }
#custom-filter-input { flex-grow: 1; padding: 6px 8px; border: 1px solid #ccc; border-radius: 4px; font-size: 12px; box-sizing: border-box; }
#btn-sort { background: #7f8c8d !important; white-space: nowrap; border:none; color:white; border-radius:4px; padding:0 10px; cursor:pointer;}
#scanner-status-bar { display:flex; justify-content:space-between; align-items:center; padding: 8px 10px; background: #f8f9fa; border-bottom: 1px solid #eee; }
#scanner-status { font-size: 12px; color: #555; font-weight: bold; }
#btn-cancel { display: none; padding: 3px 8px; background: #e74c3c; color: white; border: none; border-radius: 4px; cursor: pointer; font-size: 10px; font-weight:bold; transition: 0.2s; }
#btn-cancel:hover { background: #c0392b; }
#scanner-results { flex-grow: 1; overflow-y: auto; padding: 15px; background: #f9f9f9; display: grid; grid-template-columns: repeat(auto-fill, minmax(320px, 1fr)); gap: 15px; align-content: start; }
.scanner-result-item { background: #fff; border: 1px solid #e0e0e0; border-radius: 6px; padding: 15px; box-shadow: 0 2px 5px rgba(0,0,0,0.05); display: flex; flex-direction: column; }
.scanner-meta { display: flex; justify-content: space-between; align-items: center; margin-bottom: 10px; padding-bottom: 8px; border-bottom: 1px dashed #ddd; flex-wrap: wrap; gap: 5px; }
.scanner-source-url { font-size: 11px; color: #3498db; text-decoration: none; word-break: break-all; }
.scanner-meta-right { display: flex; align-items: center; gap: 6px; }
.scanner-matched-tags { display: flex; gap: 4px; flex-wrap: wrap; }
.matched-tag { background: #f39c12; color: #fff; padding: 2px 6px; border-radius: 4px; font-size: 10px; font-weight: bold; }
.scanner-timestamp { font-size: 11px; color: #7f8c8d; font-weight: bold; background: #ecf0f1; padding: 2px 6px; border-radius: 4px; }
.btn-delete-result { background: #e74c3c; color: white; border: none; border-radius: 4px; padding: 2px 6px; font-size: 10px; font-weight: bold; cursor: pointer; transition: 0.2s; }
.post-thumbnail { max-width: 150px; max-height: 150px; border-radius: 6px; border: 1px solid #cbd5e1; margin-top: 8px; display: block; object-fit: cover; cursor: zoom-in; }
#scroll-sentinel { grid-column: 1 / -1; height: 20px; width: 100%; }
#scanner-monitor-ui.minimized { height: 42px !important; min-height: 42px !important; width: 250px !important; min-width: 250px !important; resize: none !important; }
#scanner-monitor-ui.minimized > div:not(#scanner-header) { display: none !important; }
#scanner-image-modal { display: none; position: fixed; top: 0; left: 0; width: 100vw; height: 100vh; background: rgba(0,0,0,0.85); z-index: 9999999; justify-content: center; align-items: center; cursor: zoom-out; }
#scanner-image-modal img { max-width: 90%; max-height: 90%; border-radius: 4px; box-shadow: 0 5px 25px rgba(0,0,0,0.5); }
.author-link { text-decoration: none; color: #1e293b; display: block; margin-bottom: 4px; transition: color 0.2s; cursor: pointer; }
.author-link:hover { color: #3498db; text-decoration: underline; }
.author-avatar-link { flex-shrink: 0; display: block; width: 48px; height: 48px; cursor: pointer; }
/* Settings Modal */
#scanner-settings-modal { display:none; position:fixed; top:0; left:0; width:100vw; height:100vh; background:rgba(0,0,0,0.7); z-index:9999999; justify-content:center; align-items:center; }
.settings-card { background:#fff; padding:20px; border-radius:8px; width:450px; max-height:90vh; overflow-y:auto; display:flex; flex-direction:column; gap:12px; box-shadow: 0 10px 40px rgba(0,0,0,0.4); }
.settings-card h3 { margin:0 0 10px 0; border-bottom:1px solid #ccc; padding-bottom:10px; color:#2c3e50; }
.settings-card label { font-size:12px; font-weight:bold; color:#34495e; display:block; margin-bottom:4px; }
.settings-card textarea, .settings-card input { width:100%; box-sizing:border-box; border:1px solid #ccc; border-radius:4px; padding:6px; font-family:monospace; font-size:11px; }
</style>
<div id="scanner-header">
<span>Keyword Scanner</span>
<div style="display:flex; align-items:center; gap:10px;">
<span id="scanner-count">0 matches</span>
<button id="btn-settings" title="Settings Dashboard" style="background:none; border:none; color:white; cursor:pointer; font-size:16px; padding:0 5px; line-height:1;">⚙️</button>
<button id="btn-maximize" title="Toggle Fullscreen" style="background:none; border:none; color:white; cursor:pointer; font-weight:bold; font-size:16px; padding:0 5px; line-height:1;">⛶</button>
<button id="btn-minimize" title="Toggle Minimize" style="background:none; border:none; color:white; cursor:pointer; font-weight:bold; font-size:16px; padding:0 5px; line-height:1;">_</button>
</div>
</div>
<div id="scanner-controls">
<button id="btn-toggle" class="main-btn">Start Scan</button>
<button id="btn-clear" class="main-btn danger" title="Shift+Click for Hard Reset">Clear</button>
<button id="btn-export" class="main-btn">Export</button>
<button id="btn-import" class="main-btn">Import</button>
<div id="unified-time-panel">
<label>Start Date:</label>
<input type="datetime-local" id="scrape-start" class="tt-input" style="flex-grow:1;">
<label style="margin-left: 4px;">Limit (Days):</label>
<input type="number" id="scrape-days" class="tt-input" value="7" style="width:40px;">
</div>
</div>
<div id="scanner-filters-container">
<div id="scanner-tags-wrapper">
<div id="scanner-tags"></div><button id="btn-toggle-all-tags">All: Active</button>
</div>
<div class="filter-row">
<input type="text" id="custom-filter-input" placeholder="Type to filter...">
<button id="btn-sort">Sort: Newest</button>
</div>
</div>
<div id="scanner-status-bar">
<span id="scanner-status">Status: Idle</span>
<button id="btn-cancel">Cancel Scan</button>
</div>
<div id="scanner-results"><div id="scroll-sentinel"></div></div>
<div id="scanner-image-modal">
<img src="" id="scanner-modal-img">
</div>
<div id="scanner-settings-modal">
<div class="settings-card">
<h3>⚙️ Engine Configuration</h3>
<div>
<label>Capture Keywords (comma separated, wrap in "quotes" for exact spaces)</label>
<textarea id="set-capture" style="height:40px;"></textarea>
</div>
<div>
<label>Exclude Keywords (comma separated, wrap in "quotes" for exact spaces)</label>
<textarea id="set-exclude" style="height:40px;"></textarea>
</div>
<div>
<label>Tag Mapping (Format: TagName: keyword1, " keyword2 ")</label>
<textarea id="set-tags" style="height:90px;"></textarea>
</div>
<div style="display:flex; gap:10px;">
<div style="flex:1;"><label>Max Pages</label><input type="number" id="set-maxpages"></div>
<div style="flex:1;"><label>Delay (ms)</label><input type="number" id="set-delay"></div>
<div style="flex:1;"><label>Cycle Wait (m)</label><input type="number" id="set-cycle"></div>
</div>
<div>
<label>Base URL Target</label>
<input type="text" id="set-baseurl">
</div>
<div style="display:flex; justify-content:space-between; margin-top:10px;">
<div style="display:flex; gap:10px;">
<button id="btn-settings-export" style="padding:6px 12px; cursor:pointer; background:#3498db; color:white; border:none; border-radius:4px; font-weight:bold;" title="Export current form profile">Export</button>
<button id="btn-settings-import" style="padding:6px 12px; cursor:pointer; background:#8e44ad; color:white; border:none; border-radius:4px; font-weight:bold;" title="Load a profile into the form">Import</button>
</div>
<div style="display:flex; gap:10px;">
<button id="btn-settings-cancel" style="padding:6px 12px; cursor:pointer; background:#ecf0f1; border:1px solid #ccc; border-radius:4px; font-weight:bold;">Cancel</button>
<button id="btn-settings-save" style="padding:6px 12px; cursor:pointer; background:#27ae60; color:white; border:none; border-radius:4px; font-weight:bold;">Save & Apply</button>
</div>
</div>
</div>
</div>
`;
document.body.appendChild(ui);
// ==========================================
// STRICT UI SYNC CONTROLLER
// ==========================================
const btnToggle = document.getElementById('btn-toggle');
const btnCancel = document.getElementById('btn-cancel');
const statusEl = document.getElementById('scanner-status');
function syncUI() {
if (state.dbCorrupted) {
btnToggle.textContent = 'LOCKED'; btnToggle.disabled = true; btnToggle.style.background = '#7f8c8d'; return;
}
btnToggle.disabled = false;
btnToggle.classList.remove('takeover-mode');
if (state.isRunning) {
if (state.masterTabId === myTabId) {
btnToggle.textContent = 'Pause';
btnCancel.textContent = 'Cancel Scan'; btnCancel.style.display = 'block';
} else {
btnToggle.textContent = 'Take Over'; btnToggle.classList.add('takeover-mode');
btnCancel.textContent = 'Force Stop'; btnCancel.style.display = 'block';
}
} else {
if (state.currentPage > 1 || state.cycleTimer) {
btnToggle.textContent = 'Resume';
btnCancel.textContent = 'Reset to Idle'; btnCancel.style.display = 'block';
} else {
btnToggle.textContent = 'Start Scan';
btnCancel.style.display = 'none';
}
}
}
function updateStatus(text) {
statusEl.textContent = text;
if (!state.dbCorrupted) state.lastStatusText = text.replace(/^\[Sync\]\s*/, '');
}
function updateUIInputs() {
const scrapeDays = document.getElementById('scrape-days');
const scrapeStart = document.getElementById('scrape-start');
if (scrapeDays) scrapeDays.value = state.historicalDays || 7;
if (scrapeStart) {
if (state.isHistoricalScrape && state.historicalStartStr) {
scrapeStart.value = state.historicalStartStr;
} else {
scrapeStart.value = getLocalIsoString(new Date());
}
}
}
function initTagsUI() {
const container = document.getElementById('scanner-tags');
if (!container) return;
container.innerHTML = '';
Object.keys(state.config.tags).forEach(tagName => {
if(!renderState.tagStates[tagName]) renderState.tagStates[tagName] = 'active';
const span = document.createElement('span');
const current = renderState.tagStates[tagName];
span.className = `filter-tag ${current}`;
span.dataset.tag = tagName;
span.textContent = current === 'active' ? '✓ ' + tagName : (current === 'excluded' ? '✖ ' + tagName : tagName);
span.addEventListener('click', async () => {
const stateNow = renderState.tagStates[tagName];
let nextState = stateNow === 'active' ? 'neutral' : (stateNow === 'neutral' ? 'excluded' : 'active');
span.className = `filter-tag ${nextState}`;
span.textContent = nextState === 'active' ? '✓ ' + tagName : (nextState === 'excluded' ? '✖ ' + tagName : tagName);
renderState.tagStates[tagName] = nextState;
await yieldThread(); applyFilters();
});
container.appendChild(span);
});
Object.keys(renderState.tagStates).forEach(k => {
if(!state.config.tags[k]) delete renderState.tagStates[k];
});
}
// ==========================================
// SETTINGS PARSER LOGIC
// ==========================================
function parseKeywords(text) {
if (!text || typeof text !== 'string') return [];
return text.split(',').map(s => {
let t = s.trim();
if (t.startsWith('"') && t.endsWith('"')) return t.slice(1, -1);
if (t.startsWith("'") && t.endsWith("'")) return t.slice(1, -1);
return t;
}).filter(s => s);
}
function stringifyKeywords(arr) {
if (!Array.isArray(arr)) return "";
return arr.map(s => {
if (typeof s !== 'string') return "";
if (/^\s|\s$/.test(s)) return `"${s}"`;
return s;
}).filter(s => s !== "").join(', ');
}
function parseTags(text) {
let tags = {};
if (!text || typeof text !== 'string') return tags;
text.split('\n').forEach(line => {
let parts = line.split(':');
if(parts.length >= 2) {
let key = parts[0].trim();
let valsStr = parts.slice(1).join(':');
let vals = parseKeywords(valsStr);
if(key && vals.length) tags[key] = vals;
}
});
return tags;
}
function stringifyTags(tagsObj) {
if (!tagsObj || typeof tagsObj !== 'object') return "";
return Object.keys(tagsObj).map(k => {
let arr = tagsObj[k];
if (!Array.isArray(arr)) arr = [];
return `${k}: ${stringifyKeywords(arr)}`;
}).join('\n');
}
// ==========================================
// SETTINGS DASHBOARD LOGIC (WITH PROFILES)
// ==========================================
const settingsModal = document.getElementById('scanner-settings-modal');
const setCapture = document.getElementById('set-capture');
const setExclude = document.getElementById('set-exclude');
const setTags = document.getElementById('set-tags');
const setMaxPages = document.getElementById('set-maxpages');
const setDelay = document.getElementById('set-delay');
const setCycle = document.getElementById('set-cycle');
const setBaseUrl = document.getElementById('set-baseurl');
document.getElementById('btn-settings').addEventListener('click', (e) => {
e.stopPropagation();
setCapture.value = stringifyKeywords(state.config.captureKeywords);
setExclude.value = stringifyKeywords(state.config.excludeKeywords);
setTags.value = stringifyTags(state.config.tags);
setMaxPages.value = state.config.maxPages;
setDelay.value = state.config.delayMs;
setCycle.value = state.config.cycleDelayMinutes;
setBaseUrl.value = state.config.baseUrl;
settingsModal.style.display = 'flex';
});
document.getElementById('btn-settings-cancel').addEventListener('click', () => {
settingsModal.style.display = 'none';
});
document.getElementById('btn-settings-export').addEventListener('click', (e) => {
const btn = e.target;
const tempConfig = {
captureKeywords: parseKeywords(setCapture.value),
excludeKeywords: parseKeywords(setExclude.value),
tags: parseTags(setTags.value),
maxPages: parseInt(setMaxPages.value) || 1000,
delayMs: parseInt(setDelay.value) || 1500,
cycleDelayMinutes: parseInt(setCycle.value) || 10,
baseUrl: setBaseUrl.value.trim() || '/activities?page=1&sort=latest'
};
const dataStr = JSON.stringify(tempConfig, null, 2);
const blob = new Blob([dataStr], { type: "application/json" });
const url = URL.createObjectURL(blob);
const a = document.createElement('a'); a.href = url;
a.download = `Scanner_Config_${new Date().toISOString().split('T')[0]}.json`;
document.body.appendChild(a); a.click(); document.body.removeChild(a); URL.revokeObjectURL(url);
btn.textContent = 'Exported!';
btn.style.background = '#27ae60';
setTimeout(() => { btn.textContent = 'Export'; btn.style.background = '#3498db'; }, 2000);
});
document.getElementById('btn-settings-import').addEventListener('click', (e) => {
const btn = e.target;
const input = document.createElement('input'); input.type = 'file'; input.accept = '.json,application/json';
input.onchange = ev => {
const file = ev.target.files[0]; if (!file) return;
const reader = new FileReader();
reader.onload = (event) => {
try {
const importedConfig = JSON.parse(event.target.result);
if (importedConfig && typeof importedConfig === 'object') {
if (importedConfig.captureKeywords) setCapture.value = stringifyKeywords(importedConfig.captureKeywords);
if (importedConfig.excludeKeywords) setExclude.value = stringifyKeywords(importedConfig.excludeKeywords);
if (importedConfig.tags) setTags.value = stringifyTags(importedConfig.tags);
if (importedConfig.maxPages) setMaxPages.value = importedConfig.maxPages;
if (importedConfig.delayMs) setDelay.value = importedConfig.delayMs;
if (importedConfig.cycleDelayMinutes) setCycle.value = importedConfig.cycleDelayMinutes;
if (importedConfig.baseUrl) setBaseUrl.value = importedConfig.baseUrl;
btn.textContent = 'Loaded!';
btn.style.background = '#27ae60';
setTimeout(() => { btn.textContent = 'Import'; btn.style.background = '#8e44ad'; }, 2000);
} else {
alert("Invalid config file format.");
}
} catch (err) { alert("Error reading config file."); console.error(err); }
};
reader.readAsText(file);
};
input.click();
});
document.getElementById('btn-settings-save').addEventListener('click', async (e) => {
const btn = e.target;
btn.textContent = 'Applying...';
await yieldThread();
state.config.captureKeywords = parseKeywords(setCapture.value);
state.config.excludeKeywords = parseKeywords(setExclude.value);
state.config.tags = parseTags(setTags.value);
state.config.maxPages = parseInt(setMaxPages.value) || 1000;
state.config.delayMs = parseInt(setDelay.value) || 1500;
state.config.cycleDelayMinutes = parseInt(setCycle.value) || 10;
state.config.baseUrl = setBaseUrl.value.trim() || '/activities?page=1&sort=latest';
state.savedResults.forEach(res => {
res.tg = getMatchedTags(res.dt.toLowerCase());
});
await saveStateAsync();
initTagsUI();
await yieldThread();
applyFilters();
syncUI();
if (!state.isRunning && !state.cycleTimer) {
updateStatus("Settings applied. " + (state.currentPage > 1 ? "Paused." : "Idle."));
}
btn.textContent = 'Save & Apply';
settingsModal.style.display = 'none';
});
// ==========================================
// UI DRAG LOGIC
// ==========================================
const header = document.getElementById('scanner-header');
let isDragging = false, startX, startY, startLeft, startTop;
header.addEventListener('mousedown', (e) => {
if (e.target.tagName === 'BUTTON' || e.target.closest('button') || ui.classList.contains('maximized')) return;
isDragging = true; startX = e.clientX; startY = e.clientY;
const rect = ui.getBoundingClientRect(); startLeft = rect.left; startTop = rect.top;
ui.style.left = startLeft + 'px'; ui.style.top = startTop + 'px';
ui.style.right = 'auto'; ui.style.bottom = 'auto'; e.preventDefault();
});
document.addEventListener('mousemove', (e) => {
if (!isDragging) return;
ui.style.left = `${startLeft + e.clientX - startX}px`; ui.style.top = `${startTop + e.clientY - startY}px`;
});
document.addEventListener('mouseup', () => { isDragging = false; });
document.getElementById('btn-minimize').addEventListener('click', (e) => {
e.stopPropagation(); ui.classList.toggle('minimized');
e.target.textContent = ui.classList.contains('minimized') ? '□' : '_';
e.target.style.transform = ui.classList.contains('minimized') ? 'translateY(-2px)' : 'translateY(-5px)';
});
document.getElementById('btn-maximize').addEventListener('click', (e) => {
e.stopPropagation(); ui.classList.toggle('maximized');
e.target.textContent = ui.classList.contains('maximized') ? '🗗' : '⛶';
});
const resultsEl = document.getElementById('scanner-results');
const countEl = document.getElementById('scanner-count');
const filterInput = document.getElementById('custom-filter-input');
const sentinel = document.getElementById('scroll-sentinel');
const imageModal = document.getElementById('scanner-image-modal');
const modalImg = document.getElementById('scanner-modal-img');
imageModal.addEventListener('click', () => { imageModal.style.display = 'none'; modalImg.src = ''; });
document.getElementById('btn-toggle-all-tags').addEventListener('click', async (e) => {
const states = Object.values(renderState.tagStates);
let nextState = states.every(s => s === 'active') ? 'neutral' : (states.every(s => s === 'neutral') ? 'excluded' : 'active');
e.target.textContent = { 'active': 'All: Active', 'neutral': 'All: Neutral', 'excluded': 'All: Excluded' }[nextState];
document.querySelectorAll('.filter-tag').forEach(span => {
span.className = `filter-tag ${nextState}`;
span.textContent = nextState === 'active' ? '✓ ' + span.dataset.tag : (nextState === 'excluded' ? '✖ ' + span.dataset.tag : span.dataset.tag);
renderState.tagStates[span.dataset.tag] = nextState;
});
await yieldThread(); applyFilters();
});
resultsEl.addEventListener('click', async (e) => {
if (e.target.classList.contains('btn-delete-result')) {
const hash = e.target.getAttribute('data-hash');
state.blacklistedSignatures.add(hash);
state.savedResults = state.savedResults.filter(res => res.h !== hash);
const itemDiv = e.target.closest('.scanner-result-item');
if (itemDiv) itemDiv.remove();
renderState.currentlyRendered--;
countEl.textContent = `${state.savedResults.length} match${state.savedResults.length === 1 ? '' : 'es'}`;
updateStatus('Item deleted and blacklisted.');
saveStateAsync();
}
else if (e.target.classList.contains('post-thumbnail')) {
modalImg.src = e.target.src;
imageModal.style.display = 'flex';
}
});
// ==========================================
// DATA IMPORT & EXPORT
// ==========================================
document.getElementById('btn-export').addEventListener('click', () => {
const exportObj = {
config: state.config,
currentPage: state.currentPage, currentUrl: state.currentUrl,
blacklistedSignatures: Array.from(state.blacklistedSignatures),
savedResults: state.savedResults, users: state.users, nextUserId: state.nextUserId
};
const dataStr = JSON.stringify(exportObj);
const blob = new Blob([dataStr], { type: "application/json" });
const url = URL.createObjectURL(blob);
const a = document.createElement('a'); a.href = url;
a.download = `Scanner_Relational_DB_${new Date().toISOString().split('T')[0]}.json`;
document.body.appendChild(a); a.click(); document.body.removeChild(a); URL.revokeObjectURL(url);
updateStatus('Relational Data exported successfully.');
});
document.getElementById('btn-import').addEventListener('click', () => {
const input = document.createElement('input'); input.type = 'file'; input.accept = '.json,application/json';
input.onchange = e => {
const file = e.target.files[0]; if (!file) return;
const reader = new FileReader();
reader.onload = async (event) => {
try {
let textResult = event.target.result;
if (textResult.startsWith("LZS|") && typeof LZString !== 'undefined') textResult = LZString.decompressFromUTF16(textResult.substring(4));
else if (textResult.startsWith("LZW2|") || textResult.startsWith("LZW|")) textResult = lzwDecompressLegacy(textResult.substring(textResult.indexOf('|') + 1));
const importedState = JSON.parse(textResult);
if (importedState.savedResults) {
state.blacklistedSignatures = new Set((importedState.blacklistedSignatures || []).map(sig => sig.length > 20 ? hashString(sig) : sig));
if (importedState.config) state.config = importedState.config;
const isModernSchema = importedState.savedResults.length > 0 && importedState.savedResults[0].u !== undefined;
if (isModernSchema) {
state.users = importedState.users || {}; state.nextUserId = importedState.nextUserId || 1;
state.savedResults = importedState.savedResults;
} else {
const migratedResults = []; state.users = {}; state.nextUserId = 1; const parser = new DOMParser();
importedState.savedResults.forEach(res => {
let authorName = res.author || "Unknown"; let avatarUrl = res.avatar || "";
let displayText = res.displayText || res.normalizedText || ""; let sourceUrl = res.sourceUrl || state.config.baseUrl; let thumbFilename = res.th || "";
if (res.html) {
const doc = parser.parseFromString(res.html, 'text/html');
const authorEl = doc.querySelector('p.mb-0 a'); if (authorEl) authorName = authorEl.textContent.trim();
const avatarEl = doc.querySelector('img.main') || doc.querySelector('.user-circle img'); if (avatarEl) avatarUrl = avatarEl.src;
const thumbEl = doc.querySelector('img.img-thumbnail'); if (thumbEl) thumbFilename = getRelativeMediaPath(thumbEl.src);
let textEl = doc.querySelector('.expanding-text') || doc.querySelector('.message-left');
if (!textEl) {
const alternateTexts = Array.from(doc.querySelectorAll('p.small.mb-0'));
textEl = alternateTexts.find(p => !p.classList.contains('text-muted') && !p.classList.contains('float-right'));
}
if (textEl) { textEl.querySelectorAll('svg, img:not(.img-thumbnail), iframe, canvas').forEach(el => el.remove()); displayText = compressHtmlPayload(textEl.innerHTML); }
} else { displayText = compressHtmlPayload(displayText); }
let avatarPath = getRelativeMediaPath(avatarUrl);
if (avatarPath && !avatarPath.includes('/')) avatarPath = 'users/' + avatarPath;
let uId = Object.keys(state.users).find(id => state.users[id].n === authorName && state.users[id].a === avatarPath);
if (!uId) { uId = state.nextUserId++; state.users[uId] = { n: authorName, a: avatarPath }; }
let pg = "1", st = "";
try {
const sUrl = new URL(sourceUrl, window.location.origin);
pg = sUrl.searchParams.get('page') || "1"; st = sUrl.searchParams.get('stamp') || "";
} catch(e) {}
migratedResults.push({
u: parseInt(uId), dt: displayText, pg: pg, st: st,
tt: res.timestampText || res.timestampMs || "Unknown", ms: res.timestampMs || 0,
h: hashString(res.normalizedText || displayText),
tg: res.matchedTags || (res.normalizedText ? getMatchedTags(res.normalizedText) : []), th: thumbFilename
});
});
state.savedResults = migratedResults;
}
state.dbCorrupted = false;
const header = document.getElementById('scanner-header');
if (header) header.style.background = '#2c3e50';
state.currentPage = 1; state.currentUrl = state.config.baseUrl;
state.isRunning = false; state.masterTabId = null;
state.isHistoricalScrape = false; state.historicalStartStr = null;
state.lastCycleTimestamp = 0;
if (state.cycleTimer) { clearInterval(state.cycleTimer); state.cycleTimer = null; }
initTagsUI();
updateUIInputs();
syncUI(); await saveStateAsync(); applyFilters();
updateStatus(`Imported ${state.savedResults.length} items successfully. Idle.`);
} else { alert("Error: Invalid backup file format."); }
} catch (err) { alert("Error reading file. Ensure it is a valid JSON backup."); console.error(err); }
};
reader.readAsText(file);
};
input.click();
});
// ==========================================
// FILTERING & RENDERING
// ==========================================
filterInput.addEventListener('input', async (e) => {
renderState.customFilterText = e.target.value.toLowerCase(); await yieldThread(); applyFilters();
});
document.getElementById('btn-sort').addEventListener('click', async (e) => {
renderState.sortOrder = renderState.sortOrder === 'desc' ? 'asc' : 'desc';
e.target.textContent = renderState.sortOrder === 'desc' ? 'Sort: Newest' : 'Sort: Oldest';
await yieldThread(); applyFilters();
});
function applyFilters() {
const txtFilter = renderState.customFilterText;
const activeTags = Object.keys(renderState.tagStates).filter(k => renderState.tagStates[k] === 'active');
const totalTagsCount = Object.keys(renderState.tagStates).length;
renderState.filteredResults = state.savedResults.filter(res => {
if (txtFilter) {
const plainText = res.dt.replace(/<[^>]+>/g, ' ').toLowerCase();
const authorName = state.users[res.u] ? state.users[res.u].n.toLowerCase() : "";
if (!plainText.includes(txtFilter) && !authorName.includes(txtFilter)) {
return false;
}
}
if (!res.tg || res.tg.length === 0) return activeTags.length === totalTagsCount || activeTags.length === 0;
if (res.tg.some(t => renderState.tagStates[t] === 'excluded')) return false;
if (activeTags.length > 0 && !res.tg.some(t => renderState.tagStates[t] === 'active')) return false;
return true;
});
renderState.filteredResults.sort((a, b) => renderState.sortOrder === 'desc' ? b.ms - a.ms : a.ms - b.ms);
Array.from(resultsEl.children).forEach(child => { if (child.id !== 'scroll-sentinel') child.remove(); });
renderState.currentlyRendered = 0;
countEl.textContent = `${renderState.filteredResults.length} match${renderState.filteredResults.length === 1 ? '' : 'es'}`;
renderNextChunk();
}
function renderNextChunk() {
if (renderState.currentlyRendered >= renderState.filteredResults.length) { sentinel.style.display = 'none'; return; }
sentinel.style.display = 'block';
const fragment = document.createDocumentFragment();
const chunk = renderState.filteredResults.slice(renderState.currentlyRendered, renderState.currentlyRendered + renderState.chunkSize);
chunk.forEach(res => {
const wrapper = document.createElement('div'); wrapper.className = 'scanner-result-item';
const user = state.users[res.u] || { n: "Unknown", a: "" };
const fullAvatarUrl = user.a ? `${SYS_CONFIG.mediaBaseUrl}${user.a}` : "";
const fullSourceUrl = `${window.location.origin}/activities?page=${res.pg}&type=country&country=UK&stamp=${encodeURIComponent(res.st)}&sort=latest`;
const fullThumbUrl = res.th ? `${SYS_CONFIG.mediaBaseUrl}${res.th}` : "";
const formattedAuthor = user.n.replace(/_/g, '-');
const authorProfileUrl = `${window.location.origin}/activities?type=user&name=${encodeURIComponent(formattedAuthor)}&stamp=${encodeURIComponent(res.st)}&sort=latest`;
let tagsHtml = res.tg && res.tg.length > 0 ? `<div class="scanner-matched-tags">${res.tg.map(tag => `<span class="matched-tag">${tag}</span>`).join('')}</div>` : '';
let thumbHtml = res.th ? `<img src="${fullThumbUrl}" class="post-thumbnail" loading="lazy" title="Click to enlarge" onerror="this.style.display='none'">` : '';
wrapper.innerHTML = `
<div class="scanner-meta">
<a class="scanner-source-url" href="${fullSourceUrl}" target="_blank">View Source Page</a>
<div class="scanner-meta-right">
${tagsHtml}
<span class="scanner-timestamp">${res.tt}</span>
<button class="btn-delete-result" data-hash="${res.h}" title="Delete and Blacklist">✖</button>
</div>
</div>
<div style="display: flex; gap: 12px; margin-top: 10px; padding: 12px; background: #f8fafc; border-radius: 6px; border: 1px solid #e2e8f0; flex-grow:1;">
<a href="${authorProfileUrl}" target="_blank" class="author-avatar-link">
<img src="${fullAvatarUrl}" loading="lazy" style="width: 100%; height: 100%; border-radius: 50%; object-fit: cover;" onerror="this.parentElement.style.display='none'">
</a>
<div style="flex-grow: 1; min-width: 0;">
<a href="${authorProfileUrl}" target="_blank" class="author-link">
<strong style="font-size: 14px;">${user.n}</strong>
</a>
<div style="font-size: 13px; color: #334155; line-height: 1.5; word-break: break-word;">
${res.dt}
</div>
${thumbHtml}
</div>
</div>`;
fragment.appendChild(wrapper);
});
resultsEl.insertBefore(fragment, sentinel);
renderState.currentlyRendered += chunk.length;
}
const observer = new IntersectionObserver((entries) => { if (entries[0].isIntersecting) renderNextChunk(); }, { root: resultsEl, rootMargin: '1200px' });
observer.observe(sentinel);
// ==========================================
// THE SCRAPING ENGINE
// ==========================================
const sleep = ms => new Promise(r => setTimeout(r, ms));
function updateStatus(text) { statusEl.textContent = `Status: ${text}`; }
function getNormalizedText(domElement) {
let clone = domElement.cloneNode(true);
clone.querySelectorAll('div.mt-1.mb-0.small, .activity-comments, script').forEach(el => el.remove());
let attributeTexts = [];
clone.querySelectorAll('[alt], [title]').forEach(el => { if (el.alt) attributeTexts.push(el.alt); if (el.title) attributeTexts.push(el.title); });
let textWithoutTags = clone.innerHTML.replace(/<[^>]+>/g, ' ');
let textArea = document.createElement("textarea"); textArea.innerHTML = textWithoutTags + " " + attributeTexts.join(" ");
return textArea.value.toLowerCase().replace(/[\u200B-\u200D\uFEFF]/g, '').replace(/\s+/g, ' ').trim();
}
async function processPage(pageUrl) {
try {
debugLog(`\n[Crawler] 🌐 Fetching Page:`, pageUrl);
const response = await fetch(pageUrl); const html = await response.text();
const parser = new DOMParser(); const doc = parser.parseFromString(html, 'text/html');
const items = doc.querySelectorAll('.activity-item');
let discoveredNextUrl = null; let reachedOverlap = false;
const moreBtn = doc.querySelector('a.btn.btn-block.btn-primary.click-loading');
if (moreBtn && moreBtn.href) discoveredNextUrl = moreBtn.href;
let upperStamp = new URL(pageUrl, window.location.origin).searchParams.get('stamp');
let upperBoundDate = upperStamp ? decodeCursorDate(upperStamp) : new Date();
let lowerBoundDate = new Date(upperBoundDate.getTime() - 3600000);
if (discoveredNextUrl) {
let lowerStamp = new URL(discoveredNextUrl, window.location.origin).searchParams.get('stamp');
if (lowerStamp) lowerBoundDate = decodeCursorDate(lowerStamp) || lowerBoundDate;
}
const upperBoundMs = upperBoundDate.getTime(); const lowerBoundMs = lowerBoundDate.getTime();
const timeRange = Math.max(0, upperBoundMs - lowerBoundMs);
if (state.masterTabId === myTabId) {
const timeString = lowerBoundDate.toLocaleString(undefined, { month: 'short', day: 'numeric', hour: '2-digit', minute:'2-digit' });
updateStatus(`Scanning page ${state.currentPage} (~${timeString})...`);
}
let newlyFoundCount = 0;
items.forEach((item, index) => {
const normalizedText = getNormalizedText(item);
const textHash = hashString(normalizedText);
const timeMs = upperBoundMs - (timeRange / items.length) * (index + 0.5);
const itemDate = new Date(timeMs);
const timeText = itemDate.toLocaleString(undefined, { month: 'short', day: 'numeric', hour: '2-digit', minute:'2-digit' });
if (timeMs > state.currentCycleNewestTimestamp) state.currentCycleNewestTimestamp = timeMs;
if (!state.isHistoricalScrape && state.lastCycleTimestamp > 0) {
if (timeMs < state.lastCycleTimestamp) reachedOverlap = true;
}
if (state.blacklistedSignatures.has(textHash)) return;
const isAlreadyCaptured = state.savedResults.some(res => res.h === textHash);
if (isAlreadyCaptured) return;
const excludeMatch = state.config.excludeKeywords.find(ex => normalizedText.includes(ex.toLowerCase()));
if (excludeMatch) return;
const captureMatch = state.config.captureKeywords.find(cap => normalizedText.includes(cap.toLowerCase()));
if (!captureMatch) return;
const authorEl = item.querySelector('p.mb-0 a'); const authorName = authorEl ? authorEl.textContent.trim() : "Unknown User";
const avatarEl = item.querySelector('img.main') || item.querySelector('.user-circle img'); const avatarUrl = avatarEl ? avatarEl.src : "";
const thumbEl = item.querySelector('img.img-thumbnail'); const thumbFilename = thumbEl ? getRelativeMediaPath(thumbEl.src) : "";
let textEl = item.querySelector('.expanding-text') || item.querySelector('.message-left');
if (!textEl) {
const alternateTexts = Array.from(item.querySelectorAll('p.small.mb-0'));
textEl = alternateTexts.find(p => !p.classList.contains('text-muted') && !p.classList.contains('float-right'));
}
const displayText = textEl ? compressHtmlPayload(textEl.innerHTML) : compressHtmlPayload(normalizedText);
const avatarPath = getRelativeMediaPath(avatarUrl);
let uId = Object.keys(state.users).find(id => state.users[id].n === authorName && state.users[id].a === avatarPath);
if (!uId) { uId = state.nextUserId++; state.users[uId] = { n: authorName, a: avatarPath }; }
let pg = "1", st = "";
try {
const secureSourceUrl = (pageUrl.includes('stamp=') || !discoveredNextUrl) ? pageUrl : (() => {
const lockedObj = new URL(pageUrl, window.location.origin);
lockedObj.searchParams.set('stamp', new URL(discoveredNextUrl, window.location.origin).searchParams.get('stamp'));
return lockedObj.toString();
})();
const sUrl = new URL(secureSourceUrl, window.location.origin);
pg = sUrl.searchParams.get('page') || "1"; st = sUrl.searchParams.get('stamp') || "";
} catch(e) {}
state.savedResults.push({
u: parseInt(uId), dt: displayText, pg: pg, st: st,
tt: timeText, ms: timeMs, h: textHash, tg: getMatchedTags(normalizedText), th: thumbFilename
});
newlyFoundCount++;
});
if (newlyFoundCount > 0) {
saveStateAsync().then(() => {
if (resultsEl.scrollTop < 50) { applyFilters(); }
else { countEl.textContent = `${state.savedResults.length} match${state.savedResults.length === 1 ? '' : 'es'} (New items found)`; }
});
}
return { nextUrl: discoveredNextUrl, reachedOverlap };
} catch (error) { console.error("[Crawler] ❌ Error fetching page:", error); return { nextUrl: null, reachedOverlap: false }; }
}
async function runScanner() {
while (state.isRunning && state.masterTabId === myTabId && !state.dbCorrupted) {
if (!state.isHistoricalScrape && state.currentPage > state.config.maxPages) break;
const { nextUrl, reachedOverlap } = await processPage(state.currentUrl);
if (!state.isRunning || state.masterTabId !== myTabId) break;
if (reachedOverlap && !state.isHistoricalScrape) {
updateStatus('Reached previously scanned posts. Finishing cycle early.');
break;
}
if (!nextUrl) {
updateStatus('Feed ended or structure changed. Pausing.');
state.isRunning = false; state.masterTabId = null; break;
}
if (state.isHistoricalScrape) {
try {
const stamp = new URL(nextUrl, window.location.origin).searchParams.get('stamp');
if (stamp) {
const nextDate = decodeCursorDate(stamp);
if (nextDate && nextDate.getTime() < state.timeLimitMs) {
updateStatus(`Done! Reached historical limit.`);
state.isRunning = false; state.masterTabId = null;
state.currentPage = 1; state.isHistoricalScrape = false;
document.getElementById('scrape-start').value = getLocalIsoString(new Date());
break;
}
}
} catch(e) {}
}
state.currentUrl = nextUrl; state.currentPage++;
await saveStateAsync();
await sleep(state.config.delayMs);
}
if (!state.isRunning || state.masterTabId !== myTabId) {
syncUI(); await saveStateAsync(); return;
}
if (state.isRunning && !state.isHistoricalScrape && state.masterTabId === myTabId) {
state.lastCycleTimestamp = state.currentCycleNewestTimestamp || state.lastCycleTimestamp;
state.currentPage = 1; state.currentUrl = state.config.baseUrl;
await saveStateAsync();
let countdown = state.config.cycleDelayMinutes * 60;
syncUI();
state.cycleTimer = setInterval(() => {
if (!state.isRunning || state.masterTabId !== myTabId || state.dbCorrupted) {
clearInterval(state.cycleTimer); state.cycleTimer = null; return;
}
countdown--;
if (countdown <= 0) {
clearInterval(state.cycleTimer); state.cycleTimer = null;
if (state.isRunning && state.masterTabId === myTabId) {
state.currentCycleNewestTimestamp = 0; runScanner();
}
} else { updateStatus(`Sleeping. Next cycle in ${countdown}s...`); }
}, 1000);
}
}
// ==========================================
// SYSTEM INITIALIZATION
// ==========================================
loadState();
initTagsUI();
updateUIInputs();
applyFilters();
if (state.isRunning && state.masterTabId !== myTabId) {
syncUI();
statusEl.textContent = `[Sync] ${state.lastStatusText}`;
} else if (state.isRunning) {
syncUI();
} else if (state.savedResults.length > 0 && !state.dbCorrupted) {
syncUI(); updateStatus(`Idle. Restored page ${state.currentPage}.`);
} else {
syncUI();
}
GM_addValueChangeListener(SYS_CONFIG.storageKey, function(key, oldValue, newValue, remote) {
if (remote && !state.dbCorrupted) {
loadState();
initTagsUI();
updateUIInputs();
if (resultsEl.scrollTop < 50) { applyFilters(); }
else { countEl.textContent = `${state.savedResults.length} match${state.savedResults.length === 1 ? '' : 'es'} (Sync updated)`; }
syncUI();
if (state.isRunning) {
if (state.masterTabId !== myTabId) statusEl.textContent = `[Sync] ${state.lastStatusText}`;
} else {
if (!statusEl.textContent.includes('Idle') && !statusEl.textContent.includes('cancelled') && !statusEl.textContent.includes('limit')) {
updateStatus(`Idle. (Paused/Stopped remotely)`);
}
}
}
});
GM_addValueChangeListener(SYS_CONFIG.storageKey + '_ping', function(key, oldValue, newValue, remote) {
if (remote && state.isRunning && state.masterTabId === myTabId && !state.dbCorrupted) saveStateAsync();
});
// ==========================================
// MAIN BUTTON CONTROLS
// ==========================================
btnToggle.addEventListener('click', async () => {
if (state.dbCorrupted) return;
btnToggle.textContent = 'Processing...'; await yieldThread();
if (state.isRunning) {
if (state.masterTabId === myTabId) {
state.isRunning = false; state.masterTabId = null;
if (state.cycleTimer) { clearInterval(state.cycleTimer); state.cycleTimer = null; }
syncUI(); updateStatus('Paused.'); await saveStateAsync();
} else {
state.masterTabId = myTabId; state.isRunning = true;
syncUI(); updateStatus('Taking over scan...'); await saveStateAsync(); runScanner();
}
} else {
if (state.currentPage === 1 && !state.cycleTimer) {
const startStr = document.getElementById('scrape-start').value;
const days = parseInt(document.getElementById('scrape-days').value) || 7;
const startObj = startStr ? new Date(startStr) : new Date();
const now = new Date();
state.historicalDays = days; state.currentCycleNewestTimestamp = 0;
if (Math.abs(now.getTime() - startObj.getTime()) < 5 * 60 * 1000) {
state.isHistoricalScrape = false; state.historicalStartStr = null;
state.currentUrl = state.config.baseUrl;
document.getElementById('scrape-start').value = getLocalIsoString(now);
} else {
state.isHistoricalScrape = true; state.historicalStartStr = startStr;
state.timeLimitMs = startObj.getTime() - (days * 24 * 60 * 60 * 1000);
let targetUrl = state.config.baseUrl.replace(/page=\d+&?/, '') + (state.config.baseUrl.includes('?') ? '&' : '?') + 'stamp=' + encodeCursor(startObj);
state.currentUrl = targetUrl;
}
}
state.masterTabId = myTabId; state.isRunning = true;
syncUI(); await saveStateAsync(); runScanner();
}
});
btnCancel.addEventListener('click', async () => {
if (state.dbCorrupted) return;
if (state.isRunning && state.masterTabId !== myTabId) {
btnCancel.textContent = 'Stopping...'; await yieldThread();
state.isRunning = false; state.masterTabId = myTabId;
state.currentPage = 1; state.currentUrl = state.config.baseUrl;
state.isHistoricalScrape = false; state.historicalStartStr = null;
document.getElementById('scrape-start').value = getLocalIsoString(new Date());
syncUI(); updateStatus('Scan stopped. Returned to Idle.');
await saveStateAsync();
return;
}
btnCancel.textContent = 'Resetting...'; await yieldThread();
state.isRunning = false; state.masterTabId = myTabId;
if (state.cycleTimer) { clearInterval(state.cycleTimer); state.cycleTimer = null; }
state.currentPage = 1; state.currentUrl = state.config.baseUrl;
state.isHistoricalScrape = false; state.historicalStartStr = null;
document.getElementById('scrape-start').value = getLocalIsoString(new Date());
syncUI(); updateStatus('Scan reset. Returned to Idle.');
await saveStateAsync();
});
document.getElementById('btn-clear').addEventListener('click', async (e) => {
const isHardReset = e.shiftKey;
if(confirm(isHardReset ? "⚠️ HARD RESET: Wipe saved results AND your Blacklist?" : "Clear saved results? (Blacklist will be saved)")) {
e.target.textContent = 'Clearing...'; await yieldThread();
state.isRunning = false; state.masterTabId = myTabId;
if (state.cycleTimer) { clearInterval(state.cycleTimer); state.cycleTimer = null; }
state.currentPage = 1; state.currentUrl = state.config.baseUrl;
state.isHistoricalScrape = false; state.historicalStartStr = null; state.lastCycleTimestamp = 0;
document.getElementById('scrape-start').value = getLocalIsoString(new Date());
state.savedResults = []; state.users = {}; state.nextUserId = 1;
if (isHardReset) state.blacklistedSignatures.clear();
state.dbCorrupted = false; const header = document.getElementById('scanner-header');
if (header) header.style.background = '#2c3e50';
document.getElementById('btn-toggle-all-tags').textContent = 'All: Active';
document.querySelectorAll('.filter-tag').forEach(span => {
span.className = 'filter-tag active'; span.textContent = '✓ ' + span.dataset.tag; renderState.tagStates[span.dataset.tag] = 'active';
});
filterInput.value = ''; renderState.customFilterText = ''; applyFilters();
syncUI(); updateStatus(isHardReset ? 'Results AND Blacklist cleared.' : 'Results cleared.');
e.target.textContent = 'Clear'; await saveStateAsync();
}
});
})();