suzanne.soy/racket
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scribblings/local-redirect: break xref-key directs into on-demand parts

Browse Source 
Further reduce the "local-rediretc.js" file size by breaking out
the table of cross-reference keys --- which is only needed for indirect
links, and is therefore used infrequently. Furthermore, break the set of
keys into multiple pieces based on a hash of the key string, and each
piece is loaded independently on demand.
This commit is contained in:
Matthew Flatt 2014-03-16 14:17:53 -06:00
parent a956918adb
commit d8d5c9ec3e
2 changed files with 186 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pkgs/racket-pkgs/racket-doc/pkg/scribblings/implementation.scrbl
View File

@ -181,9 +181,12 @@ files map documentation-directory names to specific paths. Most query
references contain a documentation-directory name and a relative path
within the directory, in which case the mapping from directory names
to paths is sufficient. Indirect links, such as those created by
@racket[(seclink #:indirect? #t ...)], embed a cross-reference key, and
so @filepath{local-redirect.js} and @filepath{local-user-redirect.js}
must also embed a part of the cross-reference database. The JavaScript
@racket[(seclink #:indirect? #t ...)], embed a cross-reference key,
and so @filepath{local-redirect.js} and
@filepath{local-user-redirect.js} must also embed a part of the
cross-reference database. (This copy of the database is broken into
multiple files, each of which is loaded on demand.) The
@filepath{local-redirect.js} and @filepath{local-user-redirect.js}
files are generated as part of the special @filepath{local-redirect}
document that is implemented by the @pkgname{racket-index} package.

239
pkgs/racket-pkgs/racket-index/scribblings/main/private/local-redirect.rkt
View File

@ -12,8 +12,7 @@
(provide make-local-redirect)
(define (rewrite-code user?)
(define prefix (if user? "user_" ""))
(define (rewrite-code prefix here-url num-bins)
@string-append|{
function |@|prefix|bsearch(str, a, start, end) {
if (start >= end)
@ -31,43 +30,118 @@
var |@|prefix|link_target_prefix = false;
function |@|prefix|convert_all_links() {
|@(make-convert-all-links prefix
"" ""
#t
#f
here-url
num-bins)
AddOnLoad(|@|prefix|convert_all_links);
}|)
(define (indent n . strs)
(define i (make-string n #\space))
(apply
string-append
(let loop ([l strs])
(cond
[(null? l) null]
[(equal? (car l) "\n") (list* "\n" i (loop (cdr l)))]
[else (cons (car l) (loop (cdr l)))]))))
(define (make-convert-all-links prefix suffix lt-suffix by-doc? by-link?
here-url num-bins)
@string-append|{
|@(if by-link?
""
@string-append|{
function hash_string(s) {
var v = 0;
for (var i = 0; i < s.length; i++) {
v = (((v << 5) - v) + s.charCodeAt(i)) & 0xFFFFFF;
}
return v;
}
function demand_load(p, callback) {
// Based on a StackOverflow answer, which cites:
// JavaScript Patterns, by Stoyan Stefanov (O’Reilly). Copyright 2010 Yahoo!, Inc., 9780596806750.
var script = document.getElementsByTagName('script')[0];
var newjs = document.createElement('script');
newjs.src = p;
if (callback) {
// IE
newjs.onreadystatechange = function () {
if (newjs.readyState === 'loaded' || newjs.readyState === 'complete') {
newjs.onreadystatechange = null;
callback();
}
};
// others
newjs.onload = callback;
}
script.parentNode.appendChild(newjs);
}
var |@|prefix|loaded_link_targets = [];
var |@|prefix|link_targets = [];
var |@|prefix|num_link_target_bins = |@(format "~a" num-bins);|@"\n"
}|)
function |@|prefix|convert_all_links|@|suffix|() {
var elements = document.getElementsByClassName("Sq");
for (var i = 0; i < elements.length; i++) {
var elem = elements[i];
var doc = elem.href.match(/doc=[^&]*/);
var rel = elem.href.match(/rel=[^&]*/);
var tag = elem.href.match(/tag=[^&]*/);
if (doc && rel) {
var pos = |@|prefix|bsearch(decodeURIComponent(doc[0].substring(4)),
|@|prefix|link_dirs,
0,
|@|prefix|link_dirs.length);
if (pos) {
var p = |@|prefix|link_dirs[pos][1];
if (|@|prefix|link_target_prefix) {
p = |@|prefix|link_target_prefix + p;
}
elem.href = p + "/" + decodeURIComponent(rel[0].substring(4));
tag = false;
}
}
|@(if by-doc?
@indent[5]|{
var doc = elem.href.match(/doc=[^&]*/);
var rel = elem.href.match(/rel=[^&]*/);
if (doc && rel) {
var pos = |@|prefix|bsearch(decodeURIComponent(doc[0].substring(4)),
|@|prefix|link_dirs,
0,
|@|prefix|link_dirs.length);
if (pos) {
var p = |@|prefix|link_dirs[pos][1];
if (|@|prefix|link_target_prefix) {
p = |@|prefix|link_target_prefix + p;
}
elem.href = p + "/" + decodeURIComponent(rel[0].substring(4));
tag = false;
}
}
}|
"")
if (tag) {
var pos = |@|prefix|bsearch(decodeURIComponent(tag[0].substring(4)),
|@|prefix|link_targets,
0,
|@|prefix|link_targets.length);
if (pos) {
var p = |@|prefix|link_targets[pos][1];
if (|@|prefix|link_target_prefix) {
p = |@|prefix|link_target_prefix + p;
}
}
|@(if by-link?
@indent[7]|{
var pos = |@|prefix|bsearch(decodeURIComponent(tag[0].substring(4)),
|@|prefix|link_targets|@|lt-suffix|,
0,
|@|prefix|link_targets|@|lt-suffix|.length);
if (pos) {
var p = |@|prefix|link_targets|@|lt-suffix|[pos][1];
if (|@|prefix|link_target_prefix) {
p = |@|prefix|link_target_prefix + p;
}
elem.href = p;
}
}|
@indent[7]|{
var v = hash_string(decodeURIComponent(tag[0].substring(4))) % |@(format "~a" num-bins);
if (!|@|prefix|loaded_link_targets[v]) {
|@|prefix|loaded_link_targets[v] = true;
var p = "|@|here-url|/local-redirect_" + v + ".js";
if (|@|prefix|link_target_prefix) {
p = |@|prefix|link_target_prefix + p;
}
demand_load(p, false);
}
}|)
}
}
}
}
AddOnLoad(|@|prefix|convert_all_links);
}|)
(define search-code
@ -81,18 +155,37 @@
}
var tag = getParameterByName("tag");
var doc = getParameterByName("doc");
var rel = getParameterByName("rel");
if (doc && rel) {
var pos = bsearch(doc, link_dirs, 0, link_dirs.length);
if (pos) {
window.onload = function() {
window.location = link_dirs[pos][1] + "/" + rel;
}
tag = false;
}
}
if (tag) {
var r = bsearch(tag, 0, link_targets.length);
if (r) {
window.onload = function() {
window.location = link_targets[r][1];
}
} else {
}
var v = hash_string(tag) % num_link_target_bins;
demand_load("local-redirect_" + v + ".js",
function() {
var r = bsearch(tag, link_targets[v], 0, link_targets[v].length);
if (r) {
window.location = link_targets[v][r][1];
}
});
}
}|)
(define (js-hash-string s)
;; Needs to be the same as a hash function used for keys in JavaScript;
;; the JavaScript implementation hash_string() above is based on
;; http://stackoverflow.com/questions/7616461/generate-a-hash-from-string-in-javascript-jquery
(for/fold ([v 0]) ([c (in-string s)])
(bitwise-and (+ (- (arithmetic-shift v 5) v) (char->integer c))
#xFFFFFF)))
(define (make-local-redirect user?)
(define main-at-user? (index-at-user?))
(list
@ -114,24 +207,58 @@
(eq? t 'mod-path))))))
(define (target? v) (and (vector? v) (= 5 (vector-length v))))
(define dest-dir (send renderer get-dest-directory #t))
(define (make-dest user?)
(define (make-dest user? [suffix ""])
(build-path dest-dir
(if user?
"local-user-redirect.js"
"local-redirect.js")))
(format (if user?
"local-user-redirect~a.js"
"local-redirect~a.js")
suffix)))
(define dest (make-dest user?))
(define alt-dest (make-dest (not user?)))
;; Whether references include user and/or main docs is determined
;; by 'depends-all-main, 'depends-all-user, or 'depends-all flag
;; in "info.rkt".
(define db
(sort (for/list ([k (in-list keys)]
#:when (tag? k)
#:when (target? (resolve-get p ri k)))
(list (send renderer tag->query-string k)
(send renderer tag->url-string ri k #:absolute? user?)))
string<?
#:key car))
(define all-db-content
(for/list ([k (in-list keys)]
#:when (tag? k)
#:when (target? (resolve-get p ri k)))
(list (send renderer tag->query-string k)
(send renderer tag->url-string ri k #:absolute? user?))))
(define (write-db o unsorted-db prefix suffix lt-suffix)
(define db (sort unsorted-db string<? #:key car))
(fprintf o "~alink_targets~a = [" prefix lt-suffix)
(for ([e (in-list db)]
[i (in-naturals)])
(fprintf o (if (zero? i) "\n" ",\n"))
(fprintf o " [~s, ~s]" (car e) (cadr e)))
(fprintf o "];\n\n"))
(define prefix (if user? "user_" ""))
(define here-url (if (or user? main-at-user?)
(url->string (path->url (find-user-doc-dir)))
"../local-redirect"))
;; Break all-db-content into 1000-entry chunks that are loaded
;; on demand, so that we don't have to load a file proportional
;; to the size of all documentation to resolve a small number
;; of indirect links.
(define num-bins (add1 (quotient (length all-db-content) 1000)))
(define bins (make-vector num-bins null))
(for ([c (in-list all-db-content)])
(define h (modulo (js-hash-string (car c)) num-bins))
(vector-set! bins h (cons c (vector-ref bins h))))
(for ([unsorted-db (in-vector bins)]
[i (in-naturals)])
(define suffix (format "_~a" i))
(define lt-suffix (format "[~a]" i))
(call-with-output-file*
(make-dest #f suffix)
#:exists 'truncate/replace
(lambda (o)
(write-db o unsorted-db prefix suffix lt-suffix)
(display (make-convert-all-links prefix suffix lt-suffix #f #t here-url 0) o)
(fprintf o "\n~aconvert_all_links~a();\n" prefix suffix))))
(call-with-output-file*
dest
#:exists 'truncate/replace
@ -161,13 +288,7 @@
(url->string (path->url e))
(format "../~a" name))))
(fprintf o "];\n\n")
(fprintf o "var ~alink_targets = [" (if user? "user_" ""))
(for ([e (in-list db)]
[i (in-naturals)])
(fprintf o (if (zero? i) "\n" ",\n"))
(fprintf o " [~s, ~s]" (car e) (cadr e)))
(fprintf o "];\n\n")
(fprintf o (rewrite-code user?))
(fprintf o (rewrite-code prefix here-url num-bins))
(newline o)))
(unless (file-exists? alt-dest)
;; make empty alternate file; in `user?` mode, this