From d8d5c9ec3ef5567d9e305f098226a0b11b273096 Mon Sep 17 00:00:00 2001 From: Matthew Flatt Date: Sun, 16 Mar 2014 14:17:53 -0600 Subject: [PATCH] scribblings/local-redirect: break xref-key directs into on-demand parts Further reduce the "local-rediretc.js" file size by breaking out the table of cross-reference keys --- which is only needed for indirect links, and is therefore used infrequently. Furthermore, break the set of keys into multiple pieces based on a hash of the key string, and each piece is loaded independently on demand. --- .../pkg/scribblings/implementation.scrbl | 9 +- .../main/private/local-redirect.rkt | 239 +++++++++++++----- 2 files changed, 186 insertions(+), 62 deletions(-) diff --git a/pkgs/racket-pkgs/racket-doc/pkg/scribblings/implementation.scrbl b/pkgs/racket-pkgs/racket-doc/pkg/scribblings/implementation.scrbl index c56b1a781a..a702c8ae98 100644 --- a/pkgs/racket-pkgs/racket-doc/pkg/scribblings/implementation.scrbl +++ b/pkgs/racket-pkgs/racket-doc/pkg/scribblings/implementation.scrbl @@ -181,9 +181,12 @@ files map documentation-directory names to specific paths. Most query references contain a documentation-directory name and a relative path within the directory, in which case the mapping from directory names to paths is sufficient. Indirect links, such as those created by -@racket[(seclink #:indirect? #t ...)], embed a cross-reference key, and -so @filepath{local-redirect.js} and @filepath{local-user-redirect.js} -must also embed a part of the cross-reference database. The JavaScript +@racket[(seclink #:indirect? #t ...)], embed a cross-reference key, +and so @filepath{local-redirect.js} and +@filepath{local-user-redirect.js} must also embed a part of the +cross-reference database. (This copy of the database is broken into +multiple files, each of which is loaded on demand.) The +@filepath{local-redirect.js} and @filepath{local-user-redirect.js} files are generated as part of the special @filepath{local-redirect} document that is implemented by the @pkgname{racket-index} package. diff --git a/pkgs/racket-pkgs/racket-index/scribblings/main/private/local-redirect.rkt b/pkgs/racket-pkgs/racket-index/scribblings/main/private/local-redirect.rkt index b3a5813a1f..dd7d3f6af5 100644 --- a/pkgs/racket-pkgs/racket-index/scribblings/main/private/local-redirect.rkt +++ b/pkgs/racket-pkgs/racket-index/scribblings/main/private/local-redirect.rkt @@ -12,8 +12,7 @@ (provide make-local-redirect) -(define (rewrite-code user?) - (define prefix (if user? "user_" "")) +(define (rewrite-code prefix here-url num-bins) @string-append|{ function |@|prefix|bsearch(str, a, start, end) { if (start >= end) @@ -31,43 +30,118 @@ var |@|prefix|link_target_prefix = false; - function |@|prefix|convert_all_links() { + |@(make-convert-all-links prefix + "" "" + #t + #f + here-url + num-bins) + + AddOnLoad(|@|prefix|convert_all_links); + }|) + +(define (indent n . strs) + (define i (make-string n #\space)) + (apply + string-append + (let loop ([l strs]) + (cond + [(null? l) null] + [(equal? (car l) "\n") (list* "\n" i (loop (cdr l)))] + [else (cons (car l) (loop (cdr l)))])))) + +(define (make-convert-all-links prefix suffix lt-suffix by-doc? by-link? + here-url num-bins) + @string-append|{ + |@(if by-link? + "" + @string-append|{ + function hash_string(s) { + var v = 0; + for (var i = 0; i < s.length; i++) { + v = (((v << 5) - v) + s.charCodeAt(i)) & 0xFFFFFF; + } + return v; + } + + function demand_load(p, callback) { + // Based on a StackOverflow answer, which cites: + // JavaScript Patterns, by Stoyan Stefanov (O’Reilly). Copyright 2010 Yahoo!, Inc., 9780596806750. + var script = document.getElementsByTagName('script')[0]; + var newjs = document.createElement('script'); + newjs.src = p; + if (callback) { + // IE + newjs.onreadystatechange = function () { + if (newjs.readyState === 'loaded' || newjs.readyState === 'complete') { + newjs.onreadystatechange = null; + callback(); + } + }; + // others + newjs.onload = callback; + } + script.parentNode.appendChild(newjs); + } + + var |@|prefix|loaded_link_targets = []; + var |@|prefix|link_targets = []; + var |@|prefix|num_link_target_bins = |@(format "~a" num-bins);|@"\n" + }|) + function |@|prefix|convert_all_links|@|suffix|() { var elements = document.getElementsByClassName("Sq"); for (var i = 0; i < elements.length; i++) { var elem = elements[i]; - var doc = elem.href.match(/doc=[^&]*/); - var rel = elem.href.match(/rel=[^&]*/); var tag = elem.href.match(/tag=[^&]*/); - if (doc && rel) { - var pos = |@|prefix|bsearch(decodeURIComponent(doc[0].substring(4)), - |@|prefix|link_dirs, - 0, - |@|prefix|link_dirs.length); - if (pos) { - var p = |@|prefix|link_dirs[pos][1]; - if (|@|prefix|link_target_prefix) { - p = |@|prefix|link_target_prefix + p; - } - elem.href = p + "/" + decodeURIComponent(rel[0].substring(4)); - tag = false; - } - } + |@(if by-doc? + @indent[5]|{ + var doc = elem.href.match(/doc=[^&]*/); + var rel = elem.href.match(/rel=[^&]*/); + if (doc && rel) { + var pos = |@|prefix|bsearch(decodeURIComponent(doc[0].substring(4)), + |@|prefix|link_dirs, + 0, + |@|prefix|link_dirs.length); + if (pos) { + var p = |@|prefix|link_dirs[pos][1]; + if (|@|prefix|link_target_prefix) { + p = |@|prefix|link_target_prefix + p; + } + elem.href = p + "/" + decodeURIComponent(rel[0].substring(4)); + tag = false; + } + } + }| + "") if (tag) { - var pos = |@|prefix|bsearch(decodeURIComponent(tag[0].substring(4)), - |@|prefix|link_targets, - 0, - |@|prefix|link_targets.length); - if (pos) { - var p = |@|prefix|link_targets[pos][1]; - if (|@|prefix|link_target_prefix) { - p = |@|prefix|link_target_prefix + p; - } - } + |@(if by-link? + @indent[7]|{ + var pos = |@|prefix|bsearch(decodeURIComponent(tag[0].substring(4)), + |@|prefix|link_targets|@|lt-suffix|, + 0, + |@|prefix|link_targets|@|lt-suffix|.length); + if (pos) { + var p = |@|prefix|link_targets|@|lt-suffix|[pos][1]; + if (|@|prefix|link_target_prefix) { + p = |@|prefix|link_target_prefix + p; + } + elem.href = p; + } + }| + @indent[7]|{ + var v = hash_string(decodeURIComponent(tag[0].substring(4))) % |@(format "~a" num-bins); + if (!|@|prefix|loaded_link_targets[v]) { + |@|prefix|loaded_link_targets[v] = true; + var p = "|@|here-url|/local-redirect_" + v + ".js"; + if (|@|prefix|link_target_prefix) { + p = |@|prefix|link_target_prefix + p; + } + demand_load(p, false); + } + }|) } - } + } } - - AddOnLoad(|@|prefix|convert_all_links); }|) (define search-code @@ -81,18 +155,37 @@ } var tag = getParameterByName("tag"); + var doc = getParameterByName("doc"); + var rel = getParameterByName("rel"); + if (doc && rel) { + var pos = bsearch(doc, link_dirs, 0, link_dirs.length); + if (pos) { + window.onload = function() { + window.location = link_dirs[pos][1] + "/" + rel; + } + tag = false; + } + } if (tag) { - var r = bsearch(tag, 0, link_targets.length); - if (r) { - window.onload = function() { - window.location = link_targets[r][1]; - } - } else { - - } + var v = hash_string(tag) % num_link_target_bins; + demand_load("local-redirect_" + v + ".js", + function() { + var r = bsearch(tag, link_targets[v], 0, link_targets[v].length); + if (r) { + window.location = link_targets[v][r][1]; + } + }); } }|) +(define (js-hash-string s) + ;; Needs to be the same as a hash function used for keys in JavaScript; + ;; the JavaScript implementation hash_string() above is based on + ;; http://stackoverflow.com/questions/7616461/generate-a-hash-from-string-in-javascript-jquery + (for/fold ([v 0]) ([c (in-string s)]) + (bitwise-and (+ (- (arithmetic-shift v 5) v) (char->integer c)) + #xFFFFFF))) + (define (make-local-redirect user?) (define main-at-user? (index-at-user?)) (list @@ -114,24 +207,58 @@ (eq? t 'mod-path)))))) (define (target? v) (and (vector? v) (= 5 (vector-length v)))) (define dest-dir (send renderer get-dest-directory #t)) - (define (make-dest user?) + (define (make-dest user? [suffix ""]) (build-path dest-dir - (if user? - "local-user-redirect.js" - "local-redirect.js"))) + (format (if user? + "local-user-redirect~a.js" + "local-redirect~a.js") + suffix))) (define dest (make-dest user?)) (define alt-dest (make-dest (not user?))) ;; Whether references include user and/or main docs is determined ;; by 'depends-all-main, 'depends-all-user, or 'depends-all flag ;; in "info.rkt". - (define db - (sort (for/list ([k (in-list keys)] - #:when (tag? k) - #:when (target? (resolve-get p ri k))) - (list (send renderer tag->query-string k) - (send renderer tag->url-string ri k #:absolute? user?))) - stringquery-string k) + (send renderer tag->url-string ri k #:absolute? user?)))) + (define (write-db o unsorted-db prefix suffix lt-suffix) + (define db (sort unsorted-db stringstring (path->url (find-user-doc-dir))) + "../local-redirect")) + + ;; Break all-db-content into 1000-entry chunks that are loaded + ;; on demand, so that we don't have to load a file proportional + ;; to the size of all documentation to resolve a small number + ;; of indirect links. + (define num-bins (add1 (quotient (length all-db-content) 1000))) + (define bins (make-vector num-bins null)) + (for ([c (in-list all-db-content)]) + (define h (modulo (js-hash-string (car c)) num-bins)) + (vector-set! bins h (cons c (vector-ref bins h)))) + (for ([unsorted-db (in-vector bins)] + [i (in-naturals)]) + (define suffix (format "_~a" i)) + (define lt-suffix (format "[~a]" i)) + (call-with-output-file* + (make-dest #f suffix) + #:exists 'truncate/replace + (lambda (o) + (write-db o unsorted-db prefix suffix lt-suffix) + (display (make-convert-all-links prefix suffix lt-suffix #f #t here-url 0) o) + (fprintf o "\n~aconvert_all_links~a();\n" prefix suffix)))) + (call-with-output-file* dest #:exists 'truncate/replace @@ -161,13 +288,7 @@ (url->string (path->url e)) (format "../~a" name)))) (fprintf o "];\n\n") - (fprintf o "var ~alink_targets = [" (if user? "user_" "")) - (for ([e (in-list db)] - [i (in-naturals)]) - (fprintf o (if (zero? i) "\n" ",\n")) - (fprintf o " [~s, ~s]" (car e) (cadr e))) - (fprintf o "];\n\n") - (fprintf o (rewrite-code user?)) + (fprintf o (rewrite-code prefix here-url num-bins)) (newline o))) (unless (file-exists? alt-dest) ;; make empty alternate file; in `user?` mode, this