// ==UserScript== // @name Export any page as Markdown // @namespace https://tweeks.io/ // @version 1.2.0 // @description Copy or download any webpage as clean Markdown (selection or full page) with Readability extraction, GFM tables, code blocks, and YAML front matter // @author Tweeks // @match *://*/* // @grant GM_registerMenuCommand // @grant GM_setClipboard // @require https://unpkg.com/turndown@7.2.0/dist/turndown.js#sha256=rjYF6werkgotGBAIrOaS7FYPps1n0uKR93y8XEMizTg= // @require https://unpkg.com/turndown-plugin-gfm@1.0.2/dist/turndown-plugin-gfm.js#sha256=z3RMwbdYDwbWTOI2pP8mMKU9OJ7M8hM6CdccpENRGRI= // @require https://unpkg.com/@mozilla/readability@0.6.0/Readability.js#sha256=NNyrPQgy0AGfApkO7WthJOAp6MMrnwxvJVBUT/jf8XQ= // @run-at document-idle // ==/UserScript== (function () { 'use strict'; var baseURI = document.baseURI; // Regex to strip non-printing / invisible Unicode characters from final output. // Catches: null bytes, control chars, soft hyphens, Arabic letter marks, // zero-width spaces, directional marks, line/paragraph separators, BOM, // interlinear annotation characters. var INVISIBLE_CHARS_RE = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f-\u009f\u00ad\u061c\u200b-\u200f\u2028\u2029\ufeff\ufff9-\ufffc]/g; // ========================================================================= // URL RESOLUTION // ========================================================================= function resolveURL(raw) { if (!raw) return ''; try { return new URL(raw, baseURI).href; } catch (_) { return raw; } } // ========================================================================= // DOM CLEANUP HELPERS // ========================================================================= function resolveURLsIn(root) { root.querySelectorAll('a[href]').forEach(function (a) { a.setAttribute('href', resolveURL(a.getAttribute('href'))); }); root.querySelectorAll('img').forEach(function (img) { var lazySrc = img.getAttribute('data-src') || img.getAttribute('data-lazy-src') || img.getAttribute('data-original'); var rawSrc = img.getAttribute('src'); var best = lazySrc || rawSrc; if (best) img.setAttribute('src', resolveURL(best)); }); } function stripNoise(root) { root.querySelectorAll( 'nav, footer, aside, header' ).forEach(function (el) { el.remove(); }); root.querySelectorAll( '[role="navigation"], [role="banner"], [role="contentinfo"], ' + '[role="complementary"], [role="search"]' ).forEach(function (el) { el.remove(); }); root.querySelectorAll( 'script, style, noscript, link, meta, svg, iframe, ' + 'input, button, select, textarea, form' ).forEach(function (el) { el.remove(); }); var all = root.querySelectorAll('*'); for (var i = all.length - 1; i >= 0; i--) { var el = all[i]; if (el.getAttribute('aria-hidden') === 'true' || el.getAttribute('hidden') !== null || el.getAttribute('style')?.includes('display:none') || el.getAttribute('style')?.includes('display: none')) { el.remove(); } } } function cleanTables(root) { root.querySelectorAll('table').forEach(function (table) { Array.from(table.children).forEach(function (child) { if (!['THEAD', 'TBODY', 'TR', 'TFOOT'].includes(child.tagName)) { child.remove(); } }); }); } // ========================================================================= // HIDDEN ELEMENT DETECTION (COMPUTED STYLES) // ========================================================================= // Many sites hide elements via CSS classes (mobile menus, overlays, // collapsed sections) rather than inline styles. Checking computed styles // on the live DOM catches all of these. We tag hidden elements with a data // attribute so the tags survive cloning, then remove tagged elements from // the clone and clean up the live DOM. var HIDDEN_ATTR = 'data-cm-hidden'; // Tags to skip — these are either expected to be "hidden" (no layout box) // or contain content we want to preserve for other rules. var SKIP_COMPUTED_CHECK = { SCRIPT: 1, STYLE: 1, NOSCRIPT: 1, MATH: 1 }; function tagHiddenElements() { var it = document.createNodeIterator( document.body, NodeFilter.SHOW_ELEMENT, function (node) { if (SKIP_COMPUTED_CHECK[node.nodeName]) { return NodeFilter.FILTER_REJECT; } var style = window.getComputedStyle(node); if (style.display === 'none' || style.visibility === 'hidden') { return NodeFilter.FILTER_ACCEPT; } return NodeFilter.FILTER_SKIP; } ); var node; var tagged = []; while ((node = it.nextNode())) { node.setAttribute(HIDDEN_ATTR, '1'); tagged.push(node); } return tagged; } function removeTaggedHidden(root) { root.querySelectorAll('[' + HIDDEN_ATTR + ']').forEach(function (el) { el.remove(); }); } function cleanupTags(tagged) { for (var i = 0; i < tagged.length; i++) { tagged[i].removeAttribute(HIDDEN_ATTR); } } // ========================================================================= // READABILITY PRE-PROCESSING // ========================================================================= // These steps prepare a cloned document for Readability, working around // known bugs and preserving information that Readability would destroy. function prepareForReadability(doc) { var body = doc.body; if (!body) return; // 1. Strip heading classNames. // Readability removes elements whose classes match "sidebar", // "comment", "social", etc. Headings sometimes carry these classes // (e.g. "sidebar-heading") and get incorrectly removed. // Ref: https://github.com/mozilla/readability/issues/807 body.querySelectorAll('h1, h2, h3, h4, h5, h6').forEach(function (h) { h.className = ''; h.outerHTML = h.outerHTML; // force re-serialization }); // 2. Remove class from element. // Prevents Readability from discarding content based on // document-level class heuristics. doc.documentElement.removeAttribute('class'); // 3. Preserve
inside

 blocks.
        //    Readability strips 
 tags, which breaks code formatting.
        //    Replace with a sentinel element that Readability ignores.
        body.querySelectorAll('pre br').forEach(function (br) {
            br.outerHTML = '';
        });

        // 4. Detect code block languages and store in element IDs.
        //    Readability strips class attributes but preserves IDs.

        // GitHub-style: highlight-text-* / highlight-source-*
        body.querySelectorAll('[class*=highlight-text],[class*=highlight-source]').forEach(function (el) {
            var match = el.className.match(/highlight-(?:text|source)-([a-z0-9]+)/);
            if (match && el.firstChild && el.firstChild.nodeName === 'PRE') {
                el.firstChild.id = 'code-lang-' + match[1];
            }
        });

        // Prism.js / highlight.js: language-*
        body.querySelectorAll('[class*=language-]').forEach(function (el) {
            var match = el.className.match(/language-([a-z0-9]+)/);
            if (match) el.id = 'code-lang-' + match[1];
        });

        // Python codehilite (Sphinx/MkDocs)
        body.querySelectorAll('.codehilite > pre').forEach(function (el) {
            if (el.firstChild && el.firstChild.nodeName !== 'CODE'
                && !el.className.includes('language')) {
                el.id = 'code-lang-text';
            }
        });
    }

    // =========================================================================
    // TURNDOWN CONFIGURATION
    // =========================================================================

    var turndownService = new TurndownService({
        hr: '---',
        headingStyle: 'atx',
        bulletListMarker: '-',
        codeBlockStyle: 'fenced',
    });

    turndownService.keep(['kbd', 'sup', 'sub', 'u', 'ins', 'del', 'small', 'big']);
    turndownService.remove(['script', 'style', 'noscript']);
    turndownService.use(turndownPluginGfm.gfm);

    // Custom list item rule that preserves ordered list start attributes
    turndownService.addRule('listItem', {
        filter: 'li',
        replacement: function (content, node, options) {
            content = content
                .replace(/^\n+/, '')
                .replace(/\n+$/, '\n')
                .replace(/\n/gm, '\n    ');

            var prefix = options.bulletListMarker + ' ';
            var parent = node.parentNode;

            if (parent.nodeName === 'OL') {
                var start = parent.getAttribute('start');
                var index = Array.prototype.indexOf.call(parent.children, node);
                prefix = (start ? Number(start) + index : index + 1) + '. ';
            }

            return (
                prefix +
                content +
                (node.nextSibling && !/\n$/.test(content) ? '\n' : '')
            );
        },
    });

    // Custom fenced code block rule with language detection and dynamic fencing.
    // Overrides the default to: restore  sentinels, read language from
    // element ID (survives Readability), and grow fence length if content
    // contains backtick sequences.
    turndownService.addRule('fencedCodeBlock', {
        filter: function (node, options) {
            return (
                options.codeBlockStyle === 'fenced'
                && node.nodeName === 'PRE'
                && node.firstChild
                && node.firstChild.nodeName === 'CODE'
            );
        },
        replacement: function (content, node, options) {
            var code = node.firstChild;

            // Restore  sentinels back to newlines
            code.innerHTML = code.innerHTML.replace(/<\/br-keep>/g, '\n');

            // Detect language from ID (set during pre-processing) or class
            var lang = '';
            var idMatch = (node.id || code.id || '').match(/^code-lang-(.+)$/);
            if (idMatch) {
                lang = idMatch[1];
            } else {
                var classMatch = (code.className || '').match(/language-([a-z0-9]+)/);
                if (classMatch) lang = classMatch[1];
            }

            var text = code.textContent;

            // Dynamic fence sizing: if content contains triple backticks,
            // increase fence length to avoid conflicts
            var fence = options.fence || '```';
            var fenceChar = fence.charAt(0);
            var fenceSize = 3;
            var re = new RegExp('^' + fenceChar + '{3,}', 'gm');
            var match;
            while ((match = re.exec(text)) !== null) {
                if (match[0].length >= fenceSize) {
                    fenceSize = match[0].length + 1;
                }
            }
            var fenceStr = fenceChar.repeat(fenceSize);

            return (
                '\n\n' + fenceStr + lang + '\n' +
                text.replace(/\n$/, '') +
                '\n' + fenceStr + '\n\n'
            );
        },
    });

    // Handle bare  blocks (no  child)
    turndownService.addRule('barePreBlock', {
        filter: function (node) {
            return (
                node.nodeName === 'PRE'
                && (!node.firstChild || node.firstChild.nodeName !== 'CODE')
            );
        },
        replacement: function (content, node) {
            node.innerHTML = node.innerHTML.replace(/<\/br-keep>/g, '\n');
            var text = node.textContent;
            return '\n\n```\n' + text.replace(/\n$/, '') + '\n```\n\n';
        },
    });

    // Convert  with  +  into ![caption](src)
    turndownService.addRule('figure', {
        filter: 'figure',
        replacement: function (content, node) {
            var img = node.querySelector('img');
            var figcaption = node.querySelector('figcaption');
            if (!img) return content;

            var src = img.getAttribute('src') || '';
            var caption = figcaption ? figcaption.textContent.trim() : (img.getAttribute('alt') || '');
            return '\n\n![' + caption + '](' + src + ')\n\n';
        },
    });

    // =========================================================================
    // MARKDOWN POST-PROCESSING
    // =========================================================================

    function postProcess(markdown) {
        return markdown.replace(INVISIBLE_CHARS_RE, '');
    }

    // =========================================================================
    // SELECTION EXTRACTION
    // =========================================================================

    function getSelectionAsHTML() {
        var selection = document.getSelection();
        if (!selection || selection.rangeCount === 0 || selection.isCollapsed) {
            return '';
        }

        var range = selection.getRangeAt(0);
        var container = range.commonAncestorContainer;
        var containerTagName = '';

        if (range.toString().trim() === container.textContent.trim()) {
            if (container instanceof Element) {
                containerTagName = container.tagName.toLowerCase();
            } else {
                containerTagName = 'p';
            }
        }

        var fragment = range.cloneContents();
        var wrapper = document.createElement('div');
        wrapper.append(fragment);

        resolveURLsIn(wrapper);
        cleanTables(wrapper);

        if (containerTagName === '') {
            return wrapper.innerHTML;
        }

        if (containerTagName === 'pre') {
            var classes = (container.parentNode || container).classList.toString();
            return '' +
                wrapper.innerHTML + '';
        }

        return '<' + containerTagName + '>' +
            wrapper.innerHTML +
            '';
    }

    // =========================================================================
    // FULL PAGE EXTRACTION
    // =========================================================================

    function cleanContentRoot(root) {
        stripNoise(root);
        cleanTables(root);
        resolveURLsIn(root);
        return root;
    }

    function findContentScope(body) {
        var main = body.querySelector('main, [role="main"]');
        if (main && main.textContent.trim().length > 0) return main;

        var articles = body.querySelectorAll('article');
        if (articles.length === 1) return articles[0];
        if (articles.length > 1) {
            var wrapper = document.createElement('div');
            articles.forEach(function (a) {
                wrapper.appendChild(a.cloneNode(true));
            });
            return wrapper;
        }

        var candidates = body.querySelectorAll(
            '#content, #main-content, #main, .content, .main-content, ' +
            '.post-content, .entry-content, .page-content'
        );
        for (var i = 0; i < candidates.length; i++) {
            if (candidates[i].textContent.trim().length > 200) {
                return candidates[i];
            }
        }

        return body;
    }

    function getPageAsHTML() {
        // Tag hidden elements in the live DOM (using computed styles),
        // then clone so the tags carry over. Clean up live DOM afterward.
        var tagged = tagHiddenElements();
        try {
            var clone = document.cloneNode(true);
            removeTaggedHidden(clone);

            // Pre-process before Readability: preserve code info, fix heading
            // classes, protect 
 in , etc.
            prepareForReadability(clone);

            var article = new Readability(clone).parse();

            if (article && article.content) {
                var div = document.createElement('div');
                div.innerHTML = article.content;
                resolveURLsIn(div);
                return div.innerHTML;
            }

            // Fallback for non-article pages
            var body = document.body.cloneNode(true);
            removeTaggedHidden(body);
            var scope = findContentScope(body);
            var root = scope.cloneNode(true);
            return cleanContentRoot(root).innerHTML;
        } finally {
            cleanupTags(tagged);
        }
    }

    // =========================================================================
    // CONVERT HTML TO MARKDOWN
    // =========================================================================

    function convertToMarkdown(html) {
        return postProcess(turndownService.turndown(html));
    }

    // =========================================================================
    // TOAST (IN-PAGE FEEDBACK)
    // =========================================================================
    // Closed Shadow DOM isolates styles from the page and vice versa.
    // Fixed positioning + max z-index ensures visibility on any layout.

    var toastContainer = null;
    var toastShadow = null;
    var toastEl = null;
    var toastTimer = null;

    function initToast() {
        toastContainer = document.createElement('div');
        toastShadow = toastContainer.attachShadow({ mode: 'closed' });

        var style = document.createElement('style');
        style.textContent =
            ':host { all: initial; }' +
            '.toast {' +
            '  position: fixed;' +
            '  bottom: 24px;' +
            '  left: 50%;' +
            '  transform: translateX(-50%) translateY(0);' +
            '  z-index: 2147483647;' +
            '  background: #222;' +
            '  color: #eee;' +
            '  font-family: system-ui, -apple-system, sans-serif;' +
            '  font-size: 13px;' +
            '  line-height: 1;' +
            '  padding: 8px 14px;' +
            '  border-radius: 6px;' +
            '  box-shadow: 0 2px 8px rgba(0,0,0,0.25);' +
            '  opacity: 0;' +
            '  pointer-events: none;' +
            '  transition: opacity 0.15s ease, transform 0.15s ease;' +
            '  white-space: nowrap;' +
            '}' +
            '.toast.visible {' +
            '  opacity: 1;' +
            '}' +
            '.toast.hiding {' +
            '  opacity: 0;' +
            '  transform: translateX(-50%) translateY(4px);' +
            '}';

        toastEl = document.createElement('div');
        toastEl.className = 'toast';

        toastShadow.appendChild(style);
        toastShadow.appendChild(toastEl);
        document.body.appendChild(toastContainer);
    }

    function showToast(message) {
        if (!toastEl) initToast();

        clearTimeout(toastTimer);
        toastEl.className = 'toast';
        toastEl.textContent = message;

        // Force reflow so the transition triggers from opacity 0
        void toastEl.offsetWidth;
        toastEl.className = 'toast visible';

        toastTimer = setTimeout(function () {
            toastEl.className = 'toast hiding';
        }, 1500);
    }

    // =========================================================================
    // CLIPBOARD
    // =========================================================================

    function copyToClipboard(text, label) {
        GM_setClipboard(text, 'text');
        showToast(label + ' copied');
    }

    // =========================================================================
    // DOWNLOAD HELPERS
    // =========================================================================

    function sanitizeFilename(title) {
        return title
            .replace(/[/\\:*?"<>|]/g, '')
            .replace(/\s+/g, ' ')
            .trim()
            .substring(0, 200)
            || 'page';
    }

    function buildFrontMatter(title, url) {
        var date = new Date().toISOString().slice(0, 10);
        return '---\n'
            + 'title: "' + title.replace(/"/g, '\\"') + '"\n'
            + 'source: ' + url + '\n'
            + 'date: ' + date + '\n'
            + '---\n\n';
    }

    function downloadMarkdown(markdown) {
        var title = document.title || 'page';
        var filename = sanitizeFilename(title) + '.md';
        var content = buildFrontMatter(title, location.href) + markdown;

        var blob = new Blob([content], { type: 'text/markdown;charset=utf-8' });
        var url = URL.createObjectURL(blob);
        var a = document.createElement('a');
        a.href = url;
        a.download = filename;
        document.body.appendChild(a);
        a.click();
        a.remove();
        URL.revokeObjectURL(url);

        showToast('Downloading ' + filename);
    }

    // =========================================================================
    // MENU COMMANDS
    // =========================================================================

    GM_registerMenuCommand('Copy Selection as Markdown', function () {
        var html = getSelectionAsHTML();
        if (!html) {
            showToast('No text selected');
            return;
        }
        copyToClipboard(convertToMarkdown(html), 'Selection');
    });

    GM_registerMenuCommand('Copy Page as Markdown', function () {
        var html = getPageAsHTML();
        if (!html) {
            showToast('Page has no content');
            return;
        }
        copyToClipboard(convertToMarkdown(html), 'Page');
    });

    GM_registerMenuCommand('Download Page as Markdown', function () {
        var html = getPageAsHTML();
        if (!html) {
            showToast('Page has no content');
            return;
        }
        downloadMarkdown(convertToMarkdown(html));
    });
})();