perf: speed up hot processing (#5803)

Author: stevenjoezhang

lib/box/index.ts

@@ -141,7 +141,10 @@ class Box extends EventEmitter {
 
       // Handle deleted files
       return this._readDir(base)
-        .then(files => cacheFiles.filter(path => !files.includes(path)))
+        .then(files => {
+          const fileSet = new Set(files);
+          return cacheFiles.filter(path => !fileSet.has(path));
+        })
         .map(path => this._processFile(File.TYPE_DELETE, path));
     }).catch(err => {
       if (err && err.code !== 'ENOENT') throw err;

lib/plugins/console/generate.ts

@@ -172,8 +172,9 @@ class Generator {
       const task = (fn, path) => () => fn.call(this, path);
       const doTask = fn => fn();
       const routeList = route.list();
+      const routeSet = new Set(routeList);
       const publicFiles = Cache.filter(item => item._id.startsWith('public/')).map(item => item._id.substring(7));
-      const tasks = publicFiles.filter(path => !routeList.includes(path))
+      const tasks = publicFiles.filter(path => !routeSet.has(path))
         // Clean files
         .map(path => task(this.deleteFile, path))
         // Generate files

lib/plugins/processor/asset.ts

@@ -8,8 +8,11 @@ import type { _File } from '../../box';
 import type Hexo from '../../hexo';
 import type { Stats } from 'fs';
 import { PageSchema } from '../../types';
+import SourceIdIndex from './source_id_index';
 
 export = (ctx: Hexo) => {
+  const pageSourceIndex = new SourceIdIndex(ctx.model('Page'));
+
   return {
     pattern: new Pattern(path => {
       if (isExcludedFile(path, ctx.config)) return;
@@ -21,18 +24,18 @@ export = (ctx: Hexo) => {
 
     process: function assetProcessor(file: _File) {
       if (file.params.renderable) {
-        return processPage(ctx, file);
+        return processPage(ctx, file, pageSourceIndex);
       }
 
       return processAsset(ctx, file);
     }
   };
 };
 
-function processPage(ctx: Hexo, file: _File) {
+function processPage(ctx: Hexo, file: _File, sourceIndex: SourceIdIndex<PageSchema>) {
   const Page = ctx.model('Page');
   const { path } = file;
-  const doc = Page.findOne({source: path});
+  const doc = sourceIndex.find(path);
   const { config } = ctx;
   const { timezone } = config;
   const updated_option = config.updated_option;
@@ -43,7 +46,10 @@ function processPage(ctx: Hexo, file: _File) {
 
   if (file.type === 'delete') {
     if (doc) {
-      return doc.remove();
+      return doc.remove().then(result => {
+        sourceIndex.delete(path);
+        return result;
+      });
     }
 
     return;
@@ -106,6 +112,9 @@ function processPage(ctx: Hexo, file: _File) {
     }
 
     return Page.insert(data);
+  }).then((doc: PageSchema) => {
+    sourceIndex.set(doc);
+    return doc;
   });
 }
 

lib/plugins/processor/post.ts

@@ -10,6 +10,7 @@ import type Hexo from '../../hexo';
 import type { Stats } from 'fs';
 import { PostAssetSchema, PostSchema } from '../../types';
 import type Document from 'warehouse/dist/document';
+import SourceIdIndex from './source_id_index';
 
 const postDir = '_posts/';
 const draftDir = '_drafts/';
@@ -26,6 +27,8 @@ const preservedKeys = {
 };
 
 export = (ctx: Hexo) => {
+  const postSourceIndex = new SourceIdIndex(ctx.model('Post'));
+
   return {
     pattern: new Pattern(path => {
       if (isTmpFile(path)) return;
@@ -59,18 +62,18 @@ export = (ctx: Hexo) => {
 
     process: function postProcessor(file: _File) {
       if (file.params.renderable) {
-        return processPost(ctx, file);
+        return processPost(ctx, file, postSourceIndex);
       } else if (ctx.config.post_asset_folder) {
         return processAsset(ctx, file);
       }
     }
   };
 };
 
-function processPost(ctx: Hexo, file: _File) {
+function processPost(ctx: Hexo, file: _File, sourceIndex: SourceIdIndex<PostSchema>) {
   const Post = ctx.model('Post');
   const { path } = file.params;
-  const doc = Post.findOne({source: file.path});
+  const doc = sourceIndex.find(file.path);
   const { config } = ctx;
   const { timezone, updated_option, use_slug_as_post_title } = config;
 
@@ -82,7 +85,10 @@ function processPost(ctx: Hexo, file: _File) {
 
   if (file.type === 'delete') {
     if (doc) {
-      return doc.remove();
+      return doc.remove().then(result => {
+        sourceIndex.delete(file.path);
+        return result;
+      });
     }
 
     return;
@@ -184,11 +190,15 @@ function processPost(ctx: Hexo, file: _File) {
     }
 
     return Post.insert(data);
-  }).then((doc: PostSchema) => Promise.all([
-    doc.setCategories(categories),
-    doc.setTags(tags),
-    scanAssetDir(ctx, doc)
-  ]).then(() => markFuturePostDirty(ctx, file, doc)));
+  }).then((doc: PostSchema) => {
+    sourceIndex.set(doc);
+
+    return Promise.all([
+      doc.setCategories(categories),
+      doc.setTags(tags),
+      scanAssetDir(ctx, doc)
+    ]).then(() => markFuturePostDirty(ctx, file, doc));
+  });
 }
 
 function parseFilename(config: string, path: string) {

lib/plugins/processor/source_id_index.ts

@@ -0,0 +1,52 @@
+import type Document from 'warehouse/dist/document';
+import type Model from 'warehouse/dist/model';
+
+interface SourceSchema {
+  _id?: string;
+  source: string;
+}
+
+class SourceIdIndex<T extends SourceSchema> {
+  private readonly model: Model<T>;
+  private sourceIds?: Map<string, string>;
+
+  constructor(model: Model<T>) {
+    this.model = model;
+  }
+
+  find(source: string): Document<T> | undefined {
+    const sourceIds = this.load();
+    const id = sourceIds.get(source);
+
+    if (id) {
+      const doc = this.model.findById(id);
+      if (doc?.source === source) return doc;
+      sourceIds.delete(source);
+    }
+
+    const doc = this.model.findOne({source});
+    if (doc) this.set(doc);
+    return doc;
+  }
+
+  set(doc: T | Document<T>): void {
+    if (doc._id) this.load().set(doc.source, doc._id);
+  }
+
+  delete(source: string): void {
+    this.load().delete(source);
+  }
+
+  private load(): Map<string, string> {
+    if (!this.sourceIds) {
+      this.sourceIds = new Map();
+      this.model.forEach(doc => {
+        if (doc._id) this.sourceIds.set(doc.source, doc._id);
+      });
+    }
+
+    return this.sourceIds;
+  }
+}
+
+export default SourceIdIndex;

test/scripts/box/box.ts

@@ -229,6 +229,35 @@ describe('Box', () => {
     await rmdir(box.base);
   });
 
+  it('process() - delete only stale cache entries', async () => {
+    const box = newBox('test');
+    const existingName = 'existing.txt';
+    const deletedName = 'deleted.txt';
+    const processor = spy();
+    box.addProcessor(processor);
+
+    await BluebirdPromise.all([
+      writeFile(join(box.base, existingName), 'existing'),
+      box.Cache.insert({
+        _id: `test/${existingName}`,
+        modified: 0,
+        hash: hash('existing').toString('hex')
+      }),
+      box.Cache.insert({
+        _id: `test/${deletedName}`
+      })
+    ]);
+    await box.process();
+
+    const deletedFiles = processor.args
+      .map(([file]) => file)
+      .filter(file => file.type === 'delete');
+    deletedFiles.should.have.lengthOf(1);
+    deletedFiles[0].path.should.eql(deletedName);
+
+    await rmdir(box.base);
+  });
+
   it('process() - params', async () => {
     const box = newBox('test');
     const path = join(box.base, 'posts', '123456');

test/scripts/console/generate.ts

@@ -135,6 +135,25 @@ describe('generate', () => {
     ]);
   });
 
+  it('delete only stale generated files', async () => {
+    let routes = ['existing.txt', 'deleted.txt'];
+    hexo.extend.generator.register('stale_routes', () => routes.map(path => ({
+      path,
+      data: path
+    })));
+
+    await generate();
+    routes = ['existing.txt'];
+    await generate();
+
+    const result = await BluebirdPromise.all([
+      exists(join(hexo.public_dir, 'existing.txt')),
+      exists(join(hexo.public_dir, 'deleted.txt'))
+    ]);
+    result[0].should.be.true;
+    result[1].should.be.false;
+  });
+
   it('force regenerate', async () => {
     const src = join(hexo.source_dir, 'test.txt');
     const dest = join(hexo.public_dir, 'test.txt');

test/scripts/processors/source_id_index.ts

@@ -0,0 +1,40 @@
+import { join } from 'path';
+import { assert as sinonAssert, spy } from 'sinon';
+import Hexo from '../../../lib/hexo';
+import SourceIdIndex from '../../../lib/plugins/processor/source_id_index';
+import chai from 'chai';
+
+chai.should();
+
+describe('SourceIdIndex', () => {
+  it('uses the document id for an indexed source', async () => {
+    const hexo = new Hexo(join(__dirname, 'source_id_index_test'));
+    const Post = hexo.model('Post');
+    const doc = await Post.insert({source: '_posts/foo.md', slug: 'foo'});
+    const index = new SourceIdIndex(Post);
+    const findOne = spy(Post, 'findOne');
+
+    const result = index.find(doc.source);
+
+    result._id.should.eql(doc._id);
+    sinonAssert.notCalled(findOne);
+    findOne.restore();
+  });
+
+  it('recovers if a cached document was replaced externally', async () => {
+    const hexo = new Hexo(join(__dirname, 'source_id_index_test'));
+    const Post = hexo.model('Post');
+    const first = await Post.insert({source: '_posts/foo.md', slug: 'foo'});
+    const index = new SourceIdIndex(Post);
+    index.find(first.source)._id.should.eql(first._id);
+
+    await first.remove();
+    const second = await Post.insert({source: '_posts/foo.md', slug: 'foo'});
+    const findOne = spy(Post, 'findOne');
+
+    index.find(second.source)._id.should.eql(second._id);
+    index.find(second.source)._id.should.eql(second._id);
+    sinonAssert.calledOnce(findOne);
+    findOne.restore();
+  });
+});