User:Polygnotus/Scripts/CategoryToClipboard.js

// Wikipedia Category Items Copier - Fixed Redirect Filtering

const API_DELAY = 500; // Delay between API requests in milliseconds

const MAX_RETRIES = 3; // Maximum number of retries for failed requests

// Only run on Wikipedia category pages

if (window.location.href.includes('/wiki/Category:')) {

// Extract the category name from the URL

const categoryName = decodeURIComponent(window.location.pathname.split('/Category:')[1]);

console.log("Category Name:", categoryName);

// Create a container for our buttons

const container = document.createElement('div');

container.style.padding = '10px';

container.style.margin = '10px 0';

container.style.backgroundColor = '#f8f9fa';

container.style.border = '1px solid #a2a9b1';

container.style.borderRadius = '3px';

// Helper function to create tooltip

function addTooltip(element, text) {

element.title = text;

element.style.position = 'relative';

}

// Create the "Copy Items" button

const copyItemsBtn = document.createElement('button');

copyItemsBtn.textContent = 'Copy items';

copyItemsBtn.style.marginRight = '10px';

copyItemsBtn.style.padding = '8px 12px';

copyItemsBtn.style.cursor = 'pointer';

addTooltip(copyItemsBtn, 'Copy all items in this category. Not recursive.');

// Create the "Copy All Items" button

const copyAllItemsBtn = document.createElement('button');

copyAllItemsBtn.textContent = 'Copy items recursively';

copyAllItemsBtn.style.marginRight = '10px';

copyAllItemsBtn.style.padding = '8px 12px';

copyAllItemsBtn.style.cursor = 'pointer';

addTooltip(copyAllItemsBtn, 'Copy all items in this category AND all items in its subcategories.');

// Create the "Copy Subcats from this Category" button

const copyDirectSubcatsBtn = document.createElement('button');

copyDirectSubcatsBtn.textContent = 'Copy subcats';

copyDirectSubcatsBtn.style.marginRight = '10px';

copyDirectSubcatsBtn.style.padding = '8px 12px';

copyDirectSubcatsBtn.style.cursor = 'pointer';

addTooltip(copyDirectSubcatsBtn, 'Copy all subcategories of this category. Not recursive.');

// Create the "Copy Subcategories" button

const copySubcatsBtn = document.createElement('button');

copySubcatsBtn.textContent = 'Copy subcategories recursively';

copySubcatsBtn.style.marginRight = '10px';

copySubcatsBtn.style.padding = '8px 12px';

copySubcatsBtn.style.cursor = 'pointer';

addTooltip(copySubcatsBtn, 'Copy all subcategories of this category and its subcategories.');

// Create the "Copy Both" button

const copyBothBtn = document.createElement('button');

copyBothBtn.textContent = 'Copy both';

copyBothBtn.style.marginRight = '10px';

copyBothBtn.style.padding = '8px 12px';

copyBothBtn.style.cursor = 'pointer';

addTooltip(copyBothBtn, 'Copy all items and subcategories from this category. Not recursive.');

// Create the "Copy Both Recursively" button

const copyBothRecursiveBtn = document.createElement('button');

copyBothRecursiveBtn.textContent = 'Copy both recursively';

copyBothRecursiveBtn.style.marginRight = '10px';

copyBothRecursiveBtn.style.padding = '8px 12px';

copyBothRecursiveBtn.style.cursor = 'pointer';

addTooltip(copyBothRecursiveBtn, 'Copy all items and subcategories from this category and all its subcategories.');

// Add checkbox for URL export

const urlCheckbox = document.createElement('input');

urlCheckbox.type = 'checkbox';

urlCheckbox.id = 'includeUrls';

urlCheckbox.style.marginLeft = '15px';

const urlLabel = document.createElement('label');

urlLabel.htmlFor = 'includeUrls';

urlLabel.textContent = 'Whole URLs';

urlLabel.style.marginLeft = '5px';

addTooltip(urlLabel, 'Include full Wikipedia URLs for each item in the export');

// Create status text

const statusText = document.createElement('div');

statusText.style.marginTop = '10px';

statusText.style.color = '#555';

// Add buttons to container in the requested order

container.appendChild(copyItemsBtn);

container.appendChild(copyAllItemsBtn);

container.appendChild(copyDirectSubcatsBtn);

container.appendChild(copySubcatsBtn);

container.appendChild(copyBothBtn);

container.appendChild(copyBothRecursiveBtn);

container.appendChild(urlCheckbox);

container.appendChild(urlLabel);

container.appendChild(statusText);

// Insert container after the page title

const pageTitleHeading = document.querySelector('.mw-first-heading');

if (pageTitleHeading) {

pageTitleHeading.parentNode.insertBefore(container, pageTitleHeading.nextSibling);

} else {

document.querySelector('#content').prepend(container);

}

// Global visited set to prevent visiting any page more than once across all operations

const globalVisited = new Set();

// Function to format items with URLs if requested

function formatItems(items, includeUrls) {

if (!includeUrls) {

return items.join('\n');

}

// When URLs are requested, return ONLY the URLs, not the article names

return items.map(item => {

const encodedTitle = encodeURIComponent(item.replace(/ /g, '_'));

return `https://en.wikipedia.org/wiki/${encodedTitle}`;

}).join('\n');

}

// Function that creates a download link as an alternative to clipboard

function offerTextAsDownload(text, filename) {

// Create blob from text

const blob = new Blob([text], {type: 'text/plain'});

const url = URL.createObjectURL(blob);

// Create download link

const downloadLink = document.createElement('a');

downloadLink.href = url;

downloadLink.download = filename || 'wikipedia-category-items.txt';

downloadLink.textContent = `Download ${filename || 'items'} as text file`;

downloadLink.style.display = 'block';

downloadLink.style.marginTop = '10px';

// Add to status container

statusText.appendChild(downloadLink);

return true;

}

// Function to copy text to clipboard or offer download if copying fails

function copyToClipboardOrDownload(text, categoryName) {

return new Promise((resolve) => {

// Try to copy to clipboard first

tryClipboardCopy(text).then(success => {

if (success) {

resolve(true);

} else {

// If clipboard fails, offer download instead

const filename = `${categoryName.replace(/[^a-z0-9]/gi, '_')}-items.txt`;

// Clear the status text completely and show only the clipboard failure message

statusText.innerHTML = `

Clipboard access failed. Click the link below to download items:

`;

offerTextAsDownload(text, filename);

resolve(false);

}

});

});

}

// Try multiple clipboard methods

function tryClipboardCopy(text) {

return new Promise((resolve) => {

// First try the modern Clipboard API

if (navigator.clipboard && navigator.clipboard.writeText) {

navigator.clipboard.writeText(text)

.then(() => resolve(true))

.catch(() => {

// If Clipboard API fails, try execCommand

try {

const textarea = document.createElement('textarea');

textarea.value = text;

// Position off-screen but available

textarea.style.position = 'fixed';

textarea.style.left = '-999999px';

textarea.style.top = '-999999px';

document.body.appendChild(textarea);

textarea.focus();

textarea.select();

const success = document.execCommand('copy');

document.body.removeChild(textarea);

if (success) {

resolve(true);

} else {

resolve(false);

}

} catch (e) {

console.error("Clipboard operations failed:", e);

resolve(false);

}

});

} else {

// No clipboard API, try execCommand directly

try {

const textarea = document.createElement('textarea');

textarea.value = text;

// Position off-screen but available

textarea.style.position = 'fixed';

textarea.style.left = '-999999px';

textarea.style.top = '-999999px';

document.body.appendChild(textarea);

textarea.focus();

textarea.select();

const success = document.execCommand('copy');

document.body.removeChild(textarea);

if (success) {

resolve(true);

} else {

resolve(false);

}

} catch (e) {

console.error("Clipboard operations failed:", e);

resolve(false);

}

}

});

}

// Enhanced API request function with retry logic, rate limiting, and maxlag handling

async function makeApiRequest(url, retryCount = 0) {

try {

await new Promise(resolve => setTimeout(resolve, API_DELAY));

const response = await fetch(url);

// Handle rate limiting (HTTP 429) or server errors (5xx)

if (response.status === 429 || response.status >= 500) {

if (retryCount < MAX_RETRIES) {

const waitTime = Math.pow(2, retryCount) * 1000; // Exponential backoff

statusText.innerHTML += `
Rate limited or server error, waiting ${waitTime/1000}s before retry ${retryCount + 1}/${MAX_RETRIES}...`;

await new Promise(resolve => setTimeout(resolve, waitTime));

return makeApiRequest(url, retryCount + 1);

} else {

throw new Error(`Request failed after ${MAX_RETRIES} retries: ${response.status}`);

}

}

if (!response.ok) {

throw new Error(`HTTP ${response.status}: ${response.statusText}`);

}

const data = await response.json();

// Handle maxlag errors - these don't count as retries since they're not real failures

if (data.error && data.error.code === 'maxlag') {

const lagTime = data.error.lag || 5; // Default to 5 seconds if lag not specified

const waitTime = (lagTime + 2) * 1000; // Add 2 second buffer

statusText.innerHTML += `
Database lagged (${lagTime}s), waiting ${waitTime/1000}s before retry...`;

await new Promise(resolve => setTimeout(resolve, waitTime));

return makeApiRequest(url, retryCount); // Don't increment retry count for maxlag

}

// Handle other API errors

if (data.error) {

throw new Error(`API Error: ${data.error.code} - ${data.error.info}`);

}

return data;

} catch (error) {

if (retryCount < MAX_RETRIES) {

statusText.innerHTML += `
Request failed, retrying ${retryCount + 1}/${MAX_RETRIES}...`;

await new Promise(resolve => setTimeout(resolve, 1000));

return makeApiRequest(url, retryCount + 1);

} else {

throw error;

}

}

}

// Function to get all subcategories of a category

async function getSubcategories(categoryTitle, continueToken = null) {

try {

// Base API URL for subcategories (only get items with namespace 14, which is Category)

// Add maxlag parameter to be respectful of server load

let apiUrl = `https://en.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:${encodeURIComponent(categoryTitle)}&cmnamespace=14&cmlimit=max&maxlag=5&format=json&origin=*`;

// Add continue token if provided

if (continueToken) {

apiUrl += `&cmcontinue=${continueToken}`;

}

statusText.textContent = `Fetching subcategories for: ${categoryTitle}...`;

const data = await makeApiRequest(apiUrl);

if (!data.query || !data.query.categorymembers) {

console.error("Unexpected API response:", data);

return { subcategories: [], continueToken: null };

}

// Extract subcategories and continue token, prefix with "Category:"

const subcategories = data.query.categorymembers.map(member => member.title); // Keep full "Category:" prefix

const nextContinueToken = data.continue ? data.continue.cmcontinue : null;

return { subcategories, continueToken: nextContinueToken };

} catch (error) {

console.error("API request error:", error);

statusText.innerHTML += `
Error fetching subcategories: ${error.message}`;

return { subcategories: [], continueToken: null };

}

}

// Function to get all non-category members of a category

async function getNonCategoryMembers(categoryTitle, continueToken = null) {

try {

// Base API URL for non-category members (exclude namespace 14, which is Category)

// Add maxlag parameter to be respectful of server load

let apiUrl = `https://en.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:${encodeURIComponent(categoryTitle)}&cmnamespace=0|1|2|3|4|5|6|7|8|9|10|11|12|13|15&cmlimit=max&maxlag=5&format=json&origin=*`;

// Add continue token if provided

if (continueToken) {

apiUrl += `&cmcontinue=${continueToken}`;

}

statusText.textContent = `Fetching items for: ${categoryTitle}...`;

const data = await makeApiRequest(apiUrl);

if (!data.query || !data.query.categorymembers) {

console.error("Unexpected API response:", data);

return { members: [], continueToken: null };

}

// Extract members

const members = data.query.categorymembers.map(member => member.title);

const nextContinueToken = data.continue ? data.continue.cmcontinue : null;

return { members, continueToken: nextContinueToken };

} catch (error) {

console.error("API request error:", error);

statusText.innerHTML += `
Error fetching items: ${error.message}`;

return { members: [], continueToken: null };

}

}

// Function to get all members of a category, handling pagination

async function getAllCategoryMembers(categoryTitle) {

let allMembers = [];

let continueToken = null;

let pagesProcessed = 0;

do {

const { members, continueToken: nextToken } = await getNonCategoryMembers(categoryTitle, continueToken);

allMembers = allMembers.concat(members);

continueToken = nextToken;

pagesProcessed++;

statusText.innerHTML = `Retrieved ${allMembers.length} items from "${categoryTitle}" (page ${pagesProcessed})...`;

} while (continueToken);

return allMembers;

}

// Function to get all subcategories of a category, handling pagination

async function getAllSubcategories(categoryTitle) {

let allSubcategories = [];

let continueToken = null;

let pagesProcessed = 0;

do {

const { subcategories, continueToken: nextToken } = await getSubcategories(categoryTitle, continueToken);

allSubcategories = allSubcategories.concat(subcategories);

continueToken = nextToken;

pagesProcessed++;

} while (continueToken);

return allSubcategories;

}

// Function to recursively get all subcategories with circular reference detection

async function getAllSubcategoriesRecursive(categoryTitle) {

const visited = new Set();

const allSubcategories = [];

const queue = [`Category:${categoryTitle}`]; // Start with prefixed category

while (queue.length > 0) {

const currentCategory = queue.shift();

// Skip if already visited (circular reference detection)

if (visited.has(currentCategory) || globalVisited.has(currentCategory)) {

continue;

}

visited.add(currentCategory);

globalVisited.add(currentCategory);

statusText.innerHTML = `Exploring subcategories (found ${allSubcategories.length} categories, queue: ${queue.length})...`;

// Get direct subcategories (remove "Category:" prefix for API call)

const categoryNameForApi = currentCategory.replace('Category:', '');

const directSubcategories = await getAllSubcategories(categoryNameForApi);

// Add new subcategories to results and queue

for (const subcategory of directSubcategories) {

if (!visited.has(subcategory) && !globalVisited.has(subcategory)) {

allSubcategories.push(subcategory);

queue.push(subcategory);

}

}

}

return allSubcategories;

}

// Function to recursively get all items from a category and all its subcategories

async function getAllItemsRecursive(categoryTitle) {

const visited = new Set();

const allItems = [];

const queue = [categoryTitle]; // Start without prefix for consistency

let totalCategories = 0;

while (queue.length > 0) {

const currentCategory = queue.shift();

const categoryKey = `Category:${currentCategory}`;

// Skip if already visited (circular reference detection)

if (visited.has(categoryKey) || globalVisited.has(categoryKey)) {

continue;

}

visited.add(categoryKey);

globalVisited.add(categoryKey);

totalCategories++;

statusText.innerHTML = `Getting items from "${currentCategory}" (processed ${totalCategories} categories, found ${allItems.length} items, queue: ${queue.length})...`;

// Get items from current category

const currentItems = await getAllCategoryMembers(currentCategory);

allItems.push(...currentItems);

// Get direct subcategories and add to queue

const directSubcategories = await getAllSubcategories(currentCategory);

for (const subcategory of directSubcategories) {

if (!visited.has(subcategory) && !globalVisited.has(subcategory)) {

// Remove "Category:" prefix for queue consistency

const subcategoryName = subcategory.replace('Category:', '');

queue.push(subcategoryName);

}

}

}

return { items: allItems, totalCategories };

}

// Function to get both items and subcategories from a category (non-recursive)

async function getBothItemsAndSubcategories(categoryTitle) {

statusText.innerHTML = 'Gathering items and subcategories from this category...';

const items = await getAllCategoryMembers(categoryTitle);

const subcategories = await getAllSubcategories(categoryTitle);

return { items, subcategories };

}

// Function to recursively get both items and subcategories from a category and all its subcategories

async function getBothItemsAndSubcategoriesRecursive(categoryTitle) {

const visited = new Set();

const allItems = [];

const allSubcategories = [];

const queue = [categoryTitle]; // Start without prefix for consistency

let totalCategories = 0;

while (queue.length > 0) {

const currentCategory = queue.shift();

const categoryKey = `Category:${currentCategory}`;

// Skip if already visited (circular reference detection)

if (visited.has(categoryKey) || globalVisited.has(categoryKey)) {

continue;

}

visited.add(categoryKey);

globalVisited.add(categoryKey);

totalCategories++;

statusText.innerHTML = `Getting items and subcategories from "${currentCategory}" (processed ${totalCategories} categories, found ${allItems.length} items, ${allSubcategories.length} subcategories, queue: ${queue.length})...`;

// Get items from current category

const currentItems = await getAllCategoryMembers(currentCategory);

allItems.push(...currentItems);

// Get direct subcategories

const directSubcategories = await getAllSubcategories(currentCategory);

// Add subcategories to results and queue

for (const subcategory of directSubcategories) {

if (!visited.has(subcategory) && !globalVisited.has(subcategory)) {

allSubcategories.push(subcategory);

// Remove "Category:" prefix for queue consistency

const subcategoryName = subcategory.replace('Category:', '');

queue.push(subcategoryName);

}

}

}

return { items: allItems, subcategories: allSubcategories, totalCategories };

}

// Handle "Copy Items" button click

copyItemsBtn.addEventListener('click', async () => {

statusText.innerHTML = 'Gathering items from this category via API...';

try {

const items = await getAllCategoryMembers(categoryName);

if (items.length === 0) {

statusText.innerHTML = 'No items found in this category.';

return;

}

const includeUrls = urlCheckbox.checked;

const formattedText = formatItems(items, includeUrls);

const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName);

if (copySuccess) {

statusText.innerHTML = `Successfully copied ${items.length} items to clipboard.`;

}

} catch (error) {

statusText.innerHTML = `Error: ${error.message}`;

console.error('Error:', error);

}

});

// Handle "Copy All Items" button click

copyAllItemsBtn.addEventListener('click', async () => {

statusText.innerHTML = 'Gathering items from this category and all subcategories recursively via API (this may take a while)...';

try {

// Clear global visited set for this operation

globalVisited.clear();

// Get all items recursively

const { items: allItems, totalCategories } = await getAllItemsRecursive(categoryName);

// Deduplicate items

const uniqueItems = [...new Set(allItems)];

if (uniqueItems.length === 0) {

statusText.innerHTML = 'No items found in this category or its subcategories.';

return;

}

const includeUrls = urlCheckbox.checked;

const formattedText = formatItems(uniqueItems, includeUrls);

const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName + '_all_recursive');

if (copySuccess) {

statusText.innerHTML = `Successfully copied ${uniqueItems.length} unique items to clipboard from ${totalCategories} categories.`;

}

} catch (error) {

statusText.innerHTML = `Error: ${error.message}`;

console.error('Error:', error);

}

});

// Handle "Copy Subcats from this Category" button click

copyDirectSubcatsBtn.addEventListener('click', async () => {

statusText.innerHTML = 'Gathering direct subcategories from this category via API...';

try {

const subcategories = await getAllSubcategories(categoryName);

if (subcategories.length === 0) {

statusText.innerHTML = 'No direct subcategories found in this category.';

return;

}

const includeUrls = urlCheckbox.checked;

const formattedText = formatItems(subcategories, includeUrls);

const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName + '_direct_subcats');

if (copySuccess) {

statusText.innerHTML = `Successfully copied ${subcategories.length} direct subcategories to clipboard.`;

}

} catch (error) {

statusText.innerHTML = `Error: ${error.message}`;

console.error('Error:', error);

}

});

// Handle "Copy Subcategories" button click

copySubcatsBtn.addEventListener('click', async () => {

statusText.innerHTML = 'Gathering all subcategories recursively via API (this may take a while)...';

try {

// Clear global visited set for this operation

globalVisited.clear();

const allSubcategories = await getAllSubcategoriesRecursive(categoryName);

// Deduplicate subcategories

const uniqueSubcategories = [...new Set(allSubcategories)];

if (uniqueSubcategories.length === 0) {

statusText.innerHTML = 'No subcategories found.';

return;

}

const includeUrls = urlCheckbox.checked;

const formattedText = formatItems(uniqueSubcategories, includeUrls);

const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName + '_subcategories');

if (copySuccess) {

statusText.innerHTML = `Successfully copied ${uniqueSubcategories.length} unique subcategories to clipboard.`;

}

} catch (error) {

statusText.innerHTML = `Error: ${error.message}`;

console.error('Error:', error);

}

});

// Handle "Copy Both" button click

copyBothBtn.addEventListener('click', async () => {

statusText.innerHTML = 'Gathering both items and subcategories from this category via API...';

try {

const { items, subcategories } = await getBothItemsAndSubcategories(categoryName);

if (items.length === 0 && subcategories.length === 0) {

statusText.innerHTML = 'No items or subcategories found in this category.';

return;

}

// Combine items and subcategories

const combinedResults = [...items, ...subcategories];

const includeUrls = urlCheckbox.checked;

const formattedText = formatItems(combinedResults, includeUrls);

const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName + '_both');

if (copySuccess) {

statusText.innerHTML = `Successfully copied ${items.length} items and ${subcategories.length} subcategories (${combinedResults.length} total) to clipboard.`;

}

} catch (error) {

statusText.innerHTML = `Error: ${error.message}`;

console.error('Error:', error);

}

});

// Handle "Copy Both Recursively" button click

copyBothRecursiveBtn.addEventListener('click', async () => {

statusText.innerHTML = 'Gathering both items and subcategories recursively via API (this may take a while)...';

try {

// Clear global visited set for this operation

globalVisited.clear();

const { items: allItems, subcategories: allSubcategories, totalCategories } = await getBothItemsAndSubcategoriesRecursive(categoryName);

// Deduplicate items and subcategories

const uniqueItems = [...new Set(allItems)];

const uniqueSubcategories = [...new Set(allSubcategories)];

if (uniqueItems.length === 0 && uniqueSubcategories.length === 0) {

statusText.innerHTML = 'No items or subcategories found in this category or its subcategories.';

return;

}

// Combine items and subcategories

const combinedResults = [...uniqueItems, ...uniqueSubcategories];

const includeUrls = urlCheckbox.checked;

const formattedText = formatItems(combinedResults, includeUrls);

const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName + '_both_recursive');

if (copySuccess) {

statusText.innerHTML = `Successfully copied ${uniqueItems.length} unique items and ${uniqueSubcategories.length} unique subcategories (${combinedResults.length} total) to clipboard from ${totalCategories} categories.`;

}

} catch (error) {

statusText.innerHTML = `Error: ${error.message}`;

console.error('Error:', error);

}

});

console.log('Wikipedia Category Copier script has been loaded successfully!');

} else {

console.log('Wikipedia Category Copier: Not a category page, script inactive.');

}