wikiweb/index.html at master · NicolasWinsten/wikiweb · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>Wiki web</title>
  <style>
    body {
      margin: 0;
      overflow: hidden;
      position: relative;
    }

    #preview-popup {
      position: absolute;
      background-color: white;
      font-family: sans-serif;
      box-shadow: 0 30px 90px -20px rgba(0,0,0,0.3), 0 0 1px 1px rgba(0,0,0,0.05);
      border-radius: 2px;
      max-width: 500px;
      font-size: 14px;
    }

    #preview-popup p {
      padding: 5px;
      line-height: 1.4;
      margin-block-start: 0;
      margin-block-end: 0;
    }

    #preview-popup img {
      max-width: 294px;
      max-height: 250px;
      float: right;
      margin-left: 5px;
    }

    a {
      all: unset;
      cursor: pointer;
    }

    #search {
      position: absolute;
      left: 0px;
      top: 0px;
      width: 200px;
      font-family: sans-serif;
      font-size: 14px;
      background-color: white;
    }

    #search-box, .suggestion {
      all: unset;
      width: 100%;
      border-width: 1px;
      border-radius: 2px;
      border-color: slategray;
      border-style: solid;
      padding: 0.2em;
      background-color: white;
    }

    .suggestion {
      width: 100%;
      text-align: left;
      text-decoration: underline;
      cursor: pointer;
    }

    .suggestion:focus, .suggestion:hover {
      background-color: #cddeff;
      color: #36c;
    }
  </style>
</head>
<body>
  <div id="graph"></div>

  <a target="_blank" id="preview-popup" style="display: none;"></a>

  <div id="search">
    <input type="text" id="search-box" placeholder="type your wikipedia article">
    <div id="options-list"></div>
  </div>

  <p class="scene-nav-info" style="top:0px;color:black;">Right-click nodes to view the categories a page belongs to, Left-click to view the members of a category</p>

  <script src="https://unpkg.com/3d-force-graph"></script>
  <script type="importmap">{ "imports": { "three": "https://unpkg.com/three/build/three.module.js" }}</script>
  <script type="module">
    import SpriteText from "https://unpkg.com/three-spritetext/dist/three-spritetext.mjs"

    async function wikipediaApi(params) {
      const api = new URL("https://en.wikipedia.org/w/api.php")
      Object.entries(params).forEach(([key, val]) => api.searchParams.append(key, val))
      api.searchParams.append("origin", "*")
      api.searchParams.append("format", "json")
      api.searchParams.append("redirects", "1")
      const jsonObj = await fetch(api).then(response => response.json())
      if (jsonObj.error) throw new Error(JSON.stringify(jsonObj.error))
      return jsonObj
    }

    function notBoring(title_) {
      const title = title_.toLowerCase()
      const words = title.split(/[: \(\)]/)
      const blacklist = new Set(["stubs", "redirects", "articles", "list", "lists", "disambiguation", "portal", "portals"])
      const blacklisted = words.some(w => blacklist.has(w))

      const boringGroupings = ["year", "location", "country"]
      const boringGroup = boringGroupings.some(g => title.match(` by ${g}`))

      return !(blacklisted || boringGroup)
    }

    /**
     * return a list of pages given a search title (searches by prefix)
     */
    async function searchTitles(searchTitle) {
      const response = await wikipediaApi({
        action  : "query",
        list    : "prefixsearch",
        pssearch: searchTitle
      })
      return response.query.prefixsearch
        .map(page => page.title)
        .filter(notBoring)
    }


    /**
     * fetch the list of category pages associated with the given title
     */
    async function fetchCategories(title) {
      console.log("fetching categories of", title)
      const response = await wikipediaApi({
        action  : "query",
        prop    : "categories",
        titles  : title,
        cllimit : "max",
        clshow  : "!hidden",
      })
      return (Object.values(response.query.pages)[0].categories || [])
        .map(cat => cat.title)
        .filter(notBoring)
    }

    /**
     * fetch the members of the given category title
     */
    async function fetchMembers(category) {
      console.log("fetching members of", category)
      const response = await wikipediaApi({
        action  : "query",
        list    : "categorymembers",
        cmtitle : category,
        cmlimit : "max",
        cmnamespace  : "0|14",
      })
      return (response.query.categorymembers || [])
        .map(page => page.title)
        .filter(notBoring)
    }

    function fetchPreview(title) {
      return fetch(new URL(`https://en.wikipedia.org/api/rest_v1/page/summary/${title}`))
        .then(response => response.json())
    }

    function hide(e) {
      e.style.display = 'none'
    }

    // simple object to set a timeout for an action and also interrupt it if necessary
    class timer {
      #handle
      #running = false

      constructor(action, delay) {
        this.action = action
        this.delay = delay
      }

      launch(...params) {
        clearTimeout(this.#handle)
        this.#handle = setTimeout(() => {
          this.#running = false
          this.action(...params)
        }, this.delay)
        this.#running = true
      }

      interrupt() {
        clearTimeout(this.#handle)
        this.#running = false
      }

      get running() {
        return this.#running
      }
    }

    const preview = document.getElementById("preview-popup")
    const showPreviewTimer = new timer(displayPreview, 1000)
    const hidePreviewTimer = new timer(() => hide(preview), 300)

    preview.onmouseleave = hidePreviewTimer.launch()
    preview.onmouseenter = hidePreviewTimer.interrupt()

    /**
     * display a preview of the given page at some screen coords
     */
    async function displayPreview(title, {x, y}) {
      const response = await fetchPreview(title)
      if (response.type == "no-extract") return // nothing to display

      preview.innerHTML = ''
      preview.style.display = ''

      if (response.thumbnail) {
        const image = document.createElement("img")
        image.src = response.thumbnail.source
        preview.appendChild(image)
      }

      preview.innerHTML += response["extract_html"]

      // reset preview positions
      preview.style.left = ''
      preview.style.top = ''
      preview.style.right = ''
      preview.style.bottom = ''

      // adjust preview placement so that it doesnt fall off screen
      if (x > window.innerWidth / 2) preview.style.right = `${window.innerWidth - x}px`
      else preview.style.left = `${x}px`

      if (y > window.innerHeight / 2) preview.style.bottom = `${window.innerHeight - y}px`
      else preview.style.top = `${y}px`

      preview.href = `https://en.wikipedia.org/wiki/${title}`
    }

    const searchBox = document.getElementById("search-box")
    const suggestionsList = document.getElementById("options-list")

    function clearSuggestionsList() {
      suggestionsList.innerHTML = ''
    }

    function resetSearchBox() {
      clearSuggestionsList()
      searchBox.value = ''
    }

    const suggestArticlesTimer = new timer(suggestArticles, 500)

    // populate suggestions list based on the search box
    async function suggestArticles() {
      const results = await searchTitles(searchBox.value)
      clearSuggestionsList()
      results.forEach(title => {
        const option = document.createElement("button")
        option.innerText = title
        option.classList.add("suggestion")

        // resolve the title of the suggestion first,
        // then add it to the graph
        option.onclick = () => fetchPreview(title).then(({title}) => {
          resetSearchBox()
          clearData()
          newNode(title)
          setTimeout(() => graph.zoomToFit(1000, 300), 300)
        })

        suggestionsList.appendChild(option)
      })
    }

    searchBox.oninput = () => {
      clearSuggestionsList()
      suggestArticlesTimer.interrupt()
      if (searchBox.value.length) suggestArticlesTimer.launch()
    }

    // produce the display label for a node
    function label(node) {
      return node.title.replace(/^Category:/, "")
    }

    function isArticle({title}) {
      return !title.startsWith("Category:")
    }

    // track the titles currently existing in the graph for quick checking
    const nodeSet = new Set()

    /**
     * insert a new node into the graph simulation.
     * fetch any categories it belongs to, and check for any links to existing nodes
     */
    async function newNode(title) {
      // get all the categories this node belongs to and cross-reference it
      // with the nodes already existing in the graph
      const cats = await fetchCategories(title)
      let node = {title : title, parents : new Set(cats)}

      const newParentLinks = cats
        .filter(parent => nodeSet.has(parent))
        .map(parent => ({source : parent, target : title}))

      // check for any existing nodes that are a child of the new node
      const newChildLinks = isArticle(node) ? [] : graph.graphData().nodes
        .filter(({parents}) => parents.has(title))
        .map(child => ({source : title, target : child.title}))

      // the node is not linked to the existing graph so ignore it
      const loneNode = newParentLinks.length == 0 && newChildLinks.length == 0

      const firstNode = nodeSet.size == 0

      // do not add lone nodes
      // (sometimes redirects of titles are not resolved, so unresolved titles don't form links in the graph)
      if (!loneNode || firstNode) {
        nodeSet.add(title)
        addData([node], newParentLinks.concat(newChildLinks))
      }
    }

    /**
     * clear out the graph
     */
    function clearData() {
      nodeSet.clear()
      graph.graphData({nodes : [], links : []})
      nodeInserter.interrupt()
    }

    /**
     * add new nodes/links to the graph simulation
     */
    function addData(newNodes, newLinks) {
      const {nodes, links} = graph.graphData()
      graph.graphData({
        nodes : [...nodes, ...newNodes],
        links : [...links, ...newLinks]
      })
    }

    // nodes should be inserted one after another instead of all at once
    const nodeInserter = new timer(function(nodes) {
      const n = nodes.pop()
      if (n) {
        newNode(n).then(() => this.launch(nodes))
      }
    }, 10)

    // add in the categories to the graph that the node belongs to
    function climbPage(node) {
      if (nodeInserter.running) return

      nodeInserter.launch(
        Array.from(node.parents).filter(parent => !nodeSet.has(parent))
      )
    }

    // add in this node's category members into the graph
    async function spinPage(node) {
      if (isArticle(node)) return
      if (nodeInserter.running) return

      nodeInserter.launch(
        (await fetchMembers(node.title)).filter(child => !nodeSet.has(child))
      )
    }

    const graphDiv = document.getElementById("graph")

    const graph = ForceGraph3D()(graphDiv)
      .backgroundColor('#ffffff')
      .linkColor(() => '#000000')
      .linkCurvature(0.1)
      .nodeId("title")
      .nodeThreeObject(node => {
        const sprite = new SpriteText(label(node))
        sprite.material.depthWrite = false
        sprite.color = isArticle(node) ? '#36c' : '#000000'
        sprite.textHeight = isArticle(node) ? 5 : 5
        sprite.fontFace = isArticle(node) ? "sans-serif" : "Linux Libertine"
        return sprite
      })
      .onNodeRightClick(climbPage)
      .onNodeClick(node => {
        if (isArticle(node)) window.open(`https://en.wikipedia.org/wiki/${node.title}`, "_blank")
        else spinPage(node)
      })
      .onNodeHover(node => {
        // hovering over a node for a while should bring up a preview of the page
        showPreviewTimer.interrupt()
        if (!node) {
          hidePreviewTimer.launch()
          return
        }
        const mousePos = graph.graph2ScreenCoords(node.x, node.y, node.z)
        showPreviewTimer.launch(node.title, mousePos)
      })
      .onBackgroundClick(resetSearchBox)

    // initial node
    newNode("Spider web")

    graph.d3Force('charge').strength(-200)

    window.onresize = () => {
      graph.width(window.innerWidth)
      graph.height(window.innerHeight)
    }

    const mobile = /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent)
    if (mobile) {
      hide(graphDiv)
      hide(searchBox)
      alert("This page does not have mobile support")
    }

  </script>
</body>
</html>