integrate desktop vision

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: node:14.15
+image: node:14.19
 
 stages:
   - build
@@ -9,7 +9,7 @@ build:
   stage: build
   retry: 2
   script:
-    - npm install
+    - npm install || cat /root/.npm/_logs/*
     - node build-build-info.js
     - npm run-script build
     - echo $CI_JOB_ID > dist/CI_JOB_ID

package.json

@@ -30,7 +30,7 @@
     "express": "^4.15.3",
     "favicons-webpack-plugin": "^2.1.0",
     "file-loader": "^1.1.11",
-    "glsl-shader-loader": "git://github.com/seequent/glsl-shader-loader.git#cbb26d4921e3959570151d4179413815c36d8328",
+    "glsl-shader-loader": "git+https://github.com/seequent/glsl-shader-loader.git#cbb26d4921e3959570151d4179413815c36d8328",
     "gm": "^1.23.1",
     "html-loader": "^0.5.5",
     "html-webpack-plugin": "^3.2.0",

src/animation-3d.js

@@ -205,12 +205,15 @@ class ObjectKeyframeTracks {
     let lastFrame = frames[frames.length - 1] + 1
     let finalFrameIdx = 0
     let timesThrough = 0
-    while (finalFrameIdx <= maxFrame || !wrap) {
+    while (finalFrameIdx < maxFrame || !wrap) {
       for (let frameIdx of frames)
       {
         finalFrameIdx = (frameIdx + lastFrame * timesThrough)
+        if (finalFrameIdx >= maxFrame) finalFrameIdx = maxFrame;
         times.push(finalFrameIdx / fps)
         values.push(...valueFn(this.at(obj, frameIdx), frameIdx, finalFrameIdx / fps))
+
+        if (finalFrameIdx >= maxFrame) break;
       }
       timesThrough++
 
@@ -419,22 +422,27 @@ Util.registerComponentSystem('animation-3d', {
       let lastFrame = frames[frames.length - 1] + 1
       let finalFrameIdx = 0
       let timesThrough = 0
-      while (finalFrameIdx <= maxFrame || !wrap) {
+      while (finalFrameIdx < maxFrame || !wrap) {
         for (let frameIdx of frames)
         {
           finalFrameIdx = (frameIdx + lastFrame * timesThrough)
+          if (finalFrameIdx >= maxFrame) finalFrameIdx = maxFrame;
           times.push(finalFrameIdx / fps)
           let matrix = this.trackFrameMatrix(obj, frameIdx)
           matrix.decompose(position, rotation, scale)
           positionValues.push(...position.toArray())
           rotationValues.push(...rotation.toArray())
           scaleValues.push(...scale.toArray())
+
+          if (finalFrameIdx >= maxFrame) break;
         }
         timesThrough++
 
         if (!wrap) break;
       }
 
+      console.log("Finishing at frame", finalFrameIdx)
+
 
       let positionTrack = new THREE.VectorKeyframeTrack(`${obj.uuid}.position`, times, positionValues)
       scaleTrack = new THREE.VectorKeyframeTrack(`${obj.uuid}.scale`, times, scaleValues)
@@ -491,6 +499,7 @@ Util.registerComponentSystem('animation-3d', {
     let tracks = []
     let maxTime = 0.0
     let maxFrame = 0
+    let fps = Compositor.component.data.frameRate
     obj.traverse(o => {
       maxFrame = Math.max(maxFrame, ...this.allFrameIndices(o))
     })
@@ -499,7 +508,8 @@ Util.registerComponentSystem('animation-3d', {
       let newTracks = this.generateTHREETracks(o, {maxFrame, wrap: this.isWrapping(o)})
       if (newTracks.length <= 0) return;
 
-      maxTime = Math.max(maxTime, ...newTracks.map(t => t.times[t.times.length - 1]))
+      maxTime = maxFrame / fps
+      // maxTime = Math.max(maxTime, ...newTracks.map(t => t.times[t.times.length - 1]))
       tracks.push(...newTracks)
     })
     if (!name) name = `vartiste-${shortid.generate()}`

src/canvas-shader-processor.js

@@ -247,6 +247,19 @@ export class CanvasShaderProcessor {
       0, 0, ctx.canvas.width, ctx.canvas.height)
     ctx.globalCompositeOperation = oldOp
   }
+  apply(inputCanvas, outputCanvas = undefined)
+  {
+    if (!outputCanvas) outputCanvas = inputCanvas
+    this.setInputCanvas(inputCanvas)
+    this.update()
+    let ctx = outputCanvas.getContext('2d')
+    ctx.clearRect(0, 0, outputCanvas.width, outputCanvas.height)
+    ctx.globalCompositeOperation = 'source-over'
+    ctx.drawImage(this.canvas,
+                  0, 0, this.canvas.width, this.canvas.height,
+                  0, 0, outputCanvas.width, outputCanvas.height)
+    return outputCanvas
+  }
 }
 
 const FORWARD = new THREE.Vector3(0, 0, 1)

src/compositor.js

@@ -824,7 +824,11 @@ AFRAME.registerComponent('compositor', {
           material.needsUpdate = true
         }
 
-        if (material[layer.mode].flipY != this.data.flipY) {
+        if (layer.mode === 'matcap')
+        {
+          material[layer.mode].flipY = !this.data.flipY
+        }
+        else if (material[layer.mode].flipY != this.data.flipY) {
           material[layer.mode].flipY = this.data.flipY
         }
 

src/compression.worker.js

@@ -1,5 +1,6 @@
 import Pako from 'pako'
+import {base64ArrayBuffer} from './framework/base64ArrayBuffer.js'
 
 onmessage = function (event) {
-  postMessage(Pako.deflate(event.data));
+  postMessage("data:application/x-binary;base64," +  base64ArrayBuffer(Pako.deflate(JSON.stringify(event.data))));
 };

src/desktop-vision.js

@@ -0,0 +1,122 @@
+import { Util } from './util.js'
+import { XRControllerModelFactory } from 'three/examples/jsm/webxr/XRControllerModelFactory.js'
+import { XRHandModelFactory } from 'three/examples/jsm/webxr/XRHandModelFactory.js'
+
+Util.registerComponentSystem('desktop-vision', {
+    schema: {
+    },
+    init() {
+        /*
+            Load the Desktop Vision script asynchronously on init.
+            The sdk is then loaded immediately to manage the oauth flow
+            and automatically close Desktop Vision oauth redirect popup windows.
+        */
+
+        const script = document.createElement('script');
+        script.src = "https://js.desktop.vision/three.min.js";
+        script.onload = this.loadDesktopVision
+        document.head.appendChild(script);
+    },
+    loadDesktopVision() {
+        try {
+            window.DesktopVision.loadSDK(THREE, XRControllerModelFactory, XRHandModelFactory);
+        } catch (e) {
+            console.log(e)
+        }
+    },
+    async connectToDesktopVision() {
+        this.removeComputer();
+        const scope = encodeURIComponent("connect,list");
+        const clientID = "wG99zpg7aA2mwwmm8XHV"
+        const redirectURL = new URL(window.location.href);
+        const scene = this.el.sceneEl
+        const renderer = scene.renderer
+
+        const session = renderer.xr.getSession();
+        if (session !== null) {
+            await session.end();
+        }
+
+        redirectURL.searchParams.set("oauth", "desktopvision");
+        const redirectUri = encodeURIComponent(redirectURL);
+        window.open(`https://desktop.vision/login/?response_type=code&client_id=${clientID}&scope=${scope}&redirect_uri=${redirectUri}&redirect_type=popup&selectComputer=true`);
+
+        let roomOptionsInterval = setInterval(() => {
+            try {
+                const options = localStorage.getItem('DESKTOP_VISION_ROOM_OPTIONS')
+                localStorage.setItem("DESKTOP_VISION_ROOM_OPTIONS", null)
+                const roomOptions = JSON.parse(options)
+                if (roomOptions) {
+                    clearInterval(roomOptionsInterval)
+                    this.createComputer(roomOptions)
+                }
+            } catch (e) {
+            }
+        }, 1000);
+    },
+
+    async createComputer(roomOptions) {
+        const { ComputerConnection, Computer } = window.DesktopVision.loadSDK(THREE, XRControllerModelFactory, XRHandModelFactory);
+        const sceneContainer = document.querySelector('a-scene')
+        const parent = document.querySelector('#camera-offsetter').object3D
+
+        const scene = this.el.sceneEl
+        const camera = scene.camera
+        const renderer = scene.renderer
+
+        this.computerConnection = new ComputerConnection(roomOptions);
+        this.video = document.createElement("video");
+        this.computerConnection.on("stream-added", (newStream) => {
+            const { video, computerConnection } = this
+            video.setAttribute('webkit-playsinline', 'webkit-playsinline');
+            video.setAttribute('playsinline', 'playsinline');
+            video.srcObject = newStream;
+            video.muted = false
+            video.play();
+            const desktopOptions = {
+                renderScreenBack: true,
+                initialScalar: 1,
+                initialWidth: 2,
+                includeKeyboard: true,
+                renderAsLayer: false,
+                xrOptions: {
+                    hideControllers: true,
+                    hideHands: true,
+                    hideCursors: true,
+                    hideRay: true,
+                    parent
+                },
+            }
+            const desktop = new Computer(scene.object3D, sceneContainer, video, renderer, computerConnection, camera, desktopOptions);
+            const desktopEntity = document.createElement('a-entity')
+            scene.appendChild(desktopEntity)
+
+            desktopEntity.object3D.add(desktop);
+            desktopEntity.object3D.position.y = 2.6
+            desktopEntity.object3D.position.z = -2
+
+            this.desktopEntity = desktopEntity
+            this.desktop = desktop
+        });
+    },
+
+    removeComputer() {
+        const { computerConnection, desktop, desktopEntity } = this
+        if (!desktop) return
+        try {
+            computerConnection.disconnect()
+        } catch (e) {
+            console.log(e)
+        }
+        try {
+            desktop.destroy()
+        } catch (e) {
+            console.log(e)
+        }
+        try {
+            desktopEntity.parentNode.removeChild(desktopEntity);
+        } catch (e) {
+            console.log(e)
+        }
+    },
+})

src/index.js

@@ -55,6 +55,7 @@ require('./demo-overlay.js')
 require('./test-utilities.js')
 require('./skeletonator.js')
 require('./sketchfab.js')
+require('./desktop-vision.js')
 require('./desktop-controls.js')
 require('./about-shelf.js')
 require('./frame.js')

src/material-transformations.js

@@ -135,7 +135,10 @@ class MaterialTransformations {
       console.warn("Metalness and roughness are not same dimensions", metalness, roughness)
     }
 
-    let csp = new CanvasShaderProcessor({source: require('./shaders/combine-metal-roughness.glsl')})
+    let csp = this.csp;
+    if (!csp) {
+      csp = this.csp = new CanvasShaderProcessor({source: require('./shaders/combine-metal-roughness.glsl')})
+    }
     csp.setInputCanvas(metalness)
     csp.setCanvasAttribute('u_roughness', roughness)
     csp.update()

src/partials/toolbox-shelf.html.slm

@@ -21,6 +21,10 @@ a-entity.grab-root toolbox-shelf="" shelf="name: Experimental Functionality; clo
 
       a-entity icon-button="#asset-account-voice" popup-button="popup: kromophone-shelf;  deferred: true; scale: 0.15 0.15 0.15" tooltip="Kromophone color sonification"
       a-entity icon-button="#asset-rotate-orbit" tooltip="Start Physics" system-click-action="system: art-physics; action: startArtPhysics"
+
+    a-entity position="-1.5 1.1 0" icon-row="mergeButtons: true"
+      a-entity icon-button="#asset-dv-icon" tooltip="Connect to Desktop Vision" system-click-action="system: desktop-vision; action: connectToDesktopVision"
+      a-entity icon-button="#asset-close-circle-outline" system-click-action="system: desktop-vision; action: removeComputer" tooltip="Remove Computer"
 
 
     a-entity position="-1.5 1.1 0" icon-row="mergeButtons: true" _quick-access-row=""

src/project-file.js

@@ -6,7 +6,8 @@ import {Util} from './util.js'
 import {BrushList} from './brush-list.js'
 import {UndoStack} from './undo.js'
 import {prepareModelForExport, dedupMaterials} from './material-transformations.js'
-const FILE_VERSION = 3
+import {CanvasShaderProcessor} from './canvas-shader-processor.js'
+const FILE_VERSION = 4
 class ProjectFile {
   static update(obj) {
     if (!('_fileVersion' in obj)) obj._fileVersion = 0
@@ -61,6 +62,17 @@ class ProjectFile {
           layer.opacity = Math.pow(layer.opacity, 1/2.2)
         }
       }
+      if (obj._fileVersion < 4)
+      {
+        if (layer.mode === 'matcap')
+        {
+          let flipper = new CanvasShaderProcessor({fx: 'flip-y'})
+          for (let frame of layer.frame)
+          {
+            flipper.apply(frame)
+          }
+        }
+      }
     }
     for (let i in obj.canvases)
     {
@@ -830,8 +842,8 @@ class ProjectFile {
     let settings = compositor.el.sceneEl.systems['settings-system']
 
     return {
-      layers,
-      allNodes,
+      layers: layers.map(l => JSON.parse(JSON.stringify(l))),
+      allNodes: allNodes.map(n => n.toJSON()),
       useNodes: compositor.data.useNodes,
       width: compositor.width,
       height: compositor.height,

src/settings-system.js

@@ -197,7 +197,6 @@ Util.registerComponentSystem('settings-system', {
       compositionView.emit('updatemesh')
     }
 
-    let json = JSON.stringify(saveObj)
     if (this.data.compressProject)
     {
       console.time('compressProject')
@@ -209,18 +208,21 @@ Util.registerComponentSystem('settings-system', {
             r(message.data)
           }
           worker.onerror = e;
-          worker.postMessage(json)
+          worker.postMessage(saveObj)
         })
       }
       finally
       {
+        worker.onmessage = null
+        worker.onerror = null
         worker.terminate();
       }
       console.timeEnd('compressProject')
-      this.download("data:application/x-binary;base64," + base64ArrayBuffer(data), `${this.projectName}-${this.formatFileDate()}.vartistez`, "Project File")
+      this.download(data, `${this.projectName}-${this.formatFileDate()}.vartistez`, "Project File")
     }
     else
     {
+      let json = JSON.stringify(saveObj)
       let encoded = encodeURIComponent(json)
       this.download("data:application/x-binary," + encoded, `${this.projectName}-${this.formatFileDate()}.vartiste`, "Project File")
     }

src/template.html.slm

@@ -30,7 +30,6 @@ html
     /script src="https://aframe.io/releases/1.2.0/aframe.js"
     script src="aframe.js"
     script src="https://cdn.jsdelivr.net/npm/aframe-enviropacks@0.9.0/aframe-enviropacks.js"
-
     /script src="https://docs.opencv.org/3.4.0/opencv.js"