diff --git a/.github/workflows/proxy.yml b/.github/workflows/proxy.yml new file mode 100644 index 00000000000..e290c1f129e --- /dev/null +++ b/.github/workflows/proxy.yml @@ -0,0 +1,138 @@ +name: openresty + +on: + push: + branches: [master] + pull_request: + branches: [master] + paths: + - ".github/workflows/openresty.yml" + - "config/openresty/**" + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 # Update to latest version + + # Cache npm dependencies + - name: Cache npm packages + uses: actions/cache@v4 + with: + path: ~/.npm + key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-npm- + + # Cache luarocks + - name: Cache luarocks + uses: actions/cache@v4 + with: + path: ~/.luarocks + key: ${{ runner.os }}-luarocks-${{ hashFiles('**/*.rockspec') }} + restore-keys: | + ${{ runner.os }}-luarocks- + + - name: Install PCRE library + run: sudo apt-get install -y libpcre3 libpcre3-dev + + # Debug: Show directory structure before cleanup + - name: Debug - Show initial directory structure + run: | + pwd + ls -la + ls -la ~/.openresty || echo "No ~/.openresty directory" + ls -la /usr/local/openresty || echo "No /usr/local/openresty directory" + + # Clear previous installation + - name: Clean previous installation + run: | + sudo rm -rf ~/.openresty /usr/local/openresty + sudo mkdir -p /usr/local/openresty + sudo chown -R $USER:$USER /usr/local/openresty + + # Install OpenResty without using cache + - name: Install OpenResty + uses: leafo/gh-actions-openresty@v1 + with: + openrestyVersion: "1.21.4.2" + configureFlags: "--with-pcre-jit --with-ipv6 --with-http_v2_module" + buildCache: false + + # Debug: Verify installation + - name: Debug - Verify OpenResty installation + run: | + echo "Directory structure after installation:" + ls -la /usr/local/openresty + echo "OpenResty version:" + /usr/local/openresty/bin/openresty -v || true + echo "LuaJIT directory contents:" + ls -la /usr/local/openresty/luajit || true + + - uses: leafo/gh-actions-luarocks@v4 + with: + withLuaPath: "/usr/local/openresty/luajit/" + + - name: Install auto-ssl + run: | + which luarocks + sudo /home/runner/work/Blot/Blot/.luarocks/bin/luarocks install lua-resty-auto-ssl + sudo /home/runner/work/Blot/Blot/.luarocks/bin/luarocks list + luarocks show lua-resty-auto-ssl + + - name: Set up Redis + uses: shogo82148/actions-setup-redis@v1 + with: + redis-version: 6.2.6 + + - name: Set up Node + uses: actions/setup-node@v4 # Update to latest version + with: + node-version: 21.7.3 + cache: 'npm' # Enable built-in npm caching + + - name: Install Dependencies + run: npm ci # Use ci instead of install for more reliable builds + + - name: Create required directories + run: | + sudo mkdir -p /var/instance-ssd/logs + mkdir -p /home/runner/work/Blot/Blot/config/openresty/data/latest + + - name: Build openresty config + env: + NODE_PATH: /home/runner/work/Blot/Blot/app + NODE_SERVER_IP: 127.0.0.1 + LUA_PACKAGE_PATH: /home/runner/work/Blot/Blot/.luarocks/share/lua/5.1/?.lua;/home/runner/work/Blot/Blot/config/openresty/conf/?.lua + REDIS_IP: 127.0.0.1 + OPENRESTY_USER: runner + DISABLE_HTTP2: true + OPENRESTY_CACHE_DIRECTORY: /home/runner/work/Blot/Blot/config/openresty/data/latest/cache + OPENRESTY_CONFIG_DIRECTORY: /home/runner/work/Blot/Blot/config/openresty/data + SSL_CERTIFICATE: /home/runner/work/Blot/Blot/config/openresty/data/latest/selfsigned.crt + SSL_CERTIFICATE_KEY: /home/runner/work/Blot/Blot/config/openresty/data/latest/selfsigned.key + run: | + node config/openresty/build-config.js --skip-confirmation + cat config/openresty/data/latest/openresty.conf + + - name: Generate self-signed SSL certs + run: | + openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -keyout /home/runner/work/Blot/Blot/config/openresty/data/latest/selfsigned.key \ + -out /home/runner/work/Blot/Blot/config/openresty/data/latest/selfsigned.crt \ + -subj "/C=US/ST=Oregon/L=Portland/O=Company Name/OU=Org/CN=localhost" + + - name: Run openresty + run: sudo /usr/local/openresty/bin/openresty -c /home/runner/work/Blot/Blot/config/openresty/data/latest/openresty.conf + + - name: Verify openresty is running + run: | + curl -f localhost:80 || (sudo cat /var/instance-ssd/logs/* && exit 1) + + - name: Run openresty tests + env: + NODE_PATH: /home/runner/work/Blot/Blot/app + BLOT_OPENRESTY_TEST_USER: runner + BLOT_OPENRESTY_TEST_GROUP: runner + run: node scripts/tests config/openresty \ No newline at end of file diff --git a/proxy/Dockerfile b/proxy/Dockerfile new file mode 100644 index 00000000000..136b9720bec --- /dev/null +++ b/proxy/Dockerfile @@ -0,0 +1,44 @@ +FROM openresty/openresty:alpine-fat + +# Install dependencies with specific versions +RUN apk add --no-cache bash openssl python3 py3-pip git && \ + apk add --no-cache py3-cryptography py3-openssl + +# Install acme-nginx and create wrapper script +RUN pip3 install --no-cache-dir acme-nginx pyOpenSSL cryptography --break-system-packages && \ + echo '#!/usr/bin/env python3' > /usr/local/bin/acme-nginx && \ + echo 'from acme_nginx.client import main' >> /usr/local/bin/acme-nginx && \ + echo 'if __name__ == "__main__":' >> /usr/local/bin/acme-nginx && \ + echo ' main()' >> /usr/local/bin/acme-nginx && \ + chmod +x /usr/local/bin/acme-nginx + +# Verify acme-nginx installation +RUN acme-nginx --help + +# Create ec2-user with a fixed UID (1000) +RUN addgroup -S ec2-user && adduser -S -u 1000 -G ec2-user ec2-user + +# Install lua-resty-auto-ssl using pre-installed LuaRocks +RUN luarocks install lua-resty-auto-ssl + +# Create required directories for lua-resty-auto-ssl +RUN mkdir -p /etc/resty-auto-ssl/storage \ + /etc/resty-auto-ssl/letsencrypt/conf \ + /etc/openresty/lua /etc/ssl/private + +# Set correct permissions +RUN chmod 755 /etc/resty-auto-ssl + +# Copy pre-built configuration files +COPY data/openresty.conf /usr/local/openresty/nginx/conf/nginx.conf +COPY data/cacher.lua /etc/openresty/lua/cacher.lua + +# Copy entrypoint script +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# Expose ports +EXPOSE 80 443 + +# Set entrypoint +ENTRYPOINT ["/entrypoint.sh"] diff --git a/proxy/build/index.js b/proxy/build/index.js new file mode 100644 index 00000000000..877b7561739 --- /dev/null +++ b/proxy/build/index.js @@ -0,0 +1,195 @@ +const mustache = require("mustache"); +const config = require("config"); +const fs = require("fs-extra"); +const child_process = require("child_process"); + +function loadEnvFile() { + const envPath = require('path').join(__dirname, "..", "..", ".env"); + try { + const envContent = fs.readFileSync(envPath, "utf8"); + const envVars = envContent + .split("\n") + .filter((line) => line.trim() && !line.startsWith("#")) + .reduce((vars, line) => { + const [key, ...valueParts] = line.split("="); + const value = valueParts.join("=").trim(); + if (key && value) { + vars[key.trim()] = value.replace(/^["']|["']$/g, ""); + } + return vars; + }, {}); + + Object.assign(process.env, envVars); + } catch (error) { + console.error("Error reading .env file:", error); + } +} + +loadEnvFile(); + +const NETDATA_USER = process.env.NETDATA_USER; +const NETDATA_PASSWORD = process.env.NETDATA_PASSWORD; +const NETDATA_PORT = process.env.NETDATA_PORT; + +const NODE_SERVER_IP = process.env.NODE_SERVER_IP; +const REDIS_IP = process.env.REDIS_IP; + +if (!NODE_SERVER_IP) throw new Error("NODE_SERVER_IP not set"); +if (!REDIS_IP) throw new Error("REDIS_IP not set"); + +const OUTPUT = __dirname + "/data/latest"; +const PREVIOUS = OUTPUT + "-previous-" + Date.now(); +const CONFIG_DIRECTORY = __dirname + "/conf"; + +const template = fs.readFileSync(`${CONFIG_DIRECTORY}/server.conf`, "utf8"); +const partials = {}; + +// remote config directory on the ec2 instance to which we will copy the config files +const config_directory = + process.env.OPENRESTY_CONFIG_DIRECTORY || "/home/ec2-user/openresty"; + +// max file size for icloud uploads and webhooks bodies +// nginx requires 'M' instead of 'MB' but unfortunately +// node rawbody parser requires 'MB' instead of 'M' +// so this maps '25MB' to '25M' for nginx +const iCloud_max_body_size = `${config.icloud.maxFileSize / 1000000}M`; +const webhooks_client_max_body_size = `${ + config.webhooks.client_max_body_size / 1000000 +}M`; + +const locals = { + host: "blot.im", + blot_directory: config.blot_directory, + disable_http2: process.env.DISABLE_HTTP2, + node_ip: NODE_SERVER_IP, + node_port: "8088", + + // The maximum size of icloud uploads + iCloud_max_body_size, + + // The maximum size of webhooks bodies forwarded to the node server + webhooks_client_max_body_size, + + // used in production by the node application container running inside docker + // to communicate with the openresty cache purge endpoint on localhost + openresty_instance_private_ip: process.env.OPENRESTY_INSTANCE_PRIVATE_IP, + + // used in production if we run multiple openresty instances at the same time + server_label: process.env.SERVER_LABEL || "us", + + config_directory, + redis: { host: REDIS_IP }, + + // used only by the ci test runner since this path changes on github actions + lua_package_path: process.env.LUA_PACKAGE_PATH, + user: process.env.OPENRESTY_USER || "ec2-user", + log_directory: + process.env.OPENRESTY_LOG_DIRECTORY || "/var/instance-ssd/logs", + + // if you change the cache directory, you must also update the + // script mount-instance-store.sh + cache_directory: + process.env.OPENRESTY_CACHE_DIRECTORY || "/var/instance-ssd/cache", + ssl_certificate: + process.env.SSL_CERTIFICATE || "/etc/ssl/private/letsencrypt-domain.pem", + ssl_certificate_key: + process.env.SSL_CERTIFICATE_KEY || + "/etc/ssl/private/letsencrypt-domain.key", + + NETDATA_PASSWORD, + NETDATA_USER, + NETDATA_PORT, +}; + +// move the previous contents of the data directory to a backup +// so we can compare the new contents with the old +if (fs.existsSync(OUTPUT)) fs.moveSync(OUTPUT, PREVIOUS, { overwrite: true }); + +fs.emptyDirSync(OUTPUT); + +fs.copySync(`${__dirname}/html`, `${OUTPUT}/html`); + +fs.readdirSync(CONFIG_DIRECTORY).forEach((file) => { + // copy lua files to data directory so they are available to nginx + if (file.endsWith(".lua")) { + fs.copySync(CONFIG_DIRECTORY + "/" + file, OUTPUT + "/" + file); + } + + if (!file.endsWith(".conf")) return; + + partials[file] = fs.readFileSync(CONFIG_DIRECTORY + "/" + file, "utf8"); +}); + +const warning = ` + +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# !!!!!!!!!!! WARNING !!!!!!!!!!! +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +# Do not edit this file directly + +# This file was generated by ../build.js +# Please update the source files in ./conf and run ../build.js + +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# !!!!!!!!!!! WARNING !!!!!!!!!!! +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +`; + +const result = mustache.render(template, locals, partials); + +fs.outputFileSync(OUTPUT + "/openresty.conf", warning + result); + +// used by the proxy-tests ci action on github +if (process.argv.includes("--skip-confirmation")) { + console.log("Build complete"); + return process.exit(0); +} + +// compare the new contents with the old +const diff = child_process.spawnSync( + "/opt/homebrew/bin/diff", + ["--color", "-r", PREVIOUS, OUTPUT], + { stdio: "inherit" } +); + +if (diff.error) { + console.error(diff.error); +} else { + // ask the user to confirm the changes + // if y, exit with success + // if n, restore the previous contents to the OUTPUT directory + // and remove the PREVIOUS directory + // if anything else, ask the user to confirm again + + const readline = require("readline"); + + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + const question = () => { + rl.question("Do you want to keep these changes? [y/n] ", (answer) => { + if (answer === "y") { + console.log("Changes kept. Build complete"); + rl.close(); + } else if (answer === "n") { + console.log("Changes discarded"); + fs.removeSync(OUTPUT); + fs.moveSync(PREVIOUS, OUTPUT); + fs.removeSync(PREVIOUS); + console.log("Done"); + rl.close(); + // exit with failure + process.exit(1); + } else { + console.log("Please answer 'y' or 'n'"); + question(); + } + }); + }; + + question(); +} diff --git a/proxy/build/loadEnvFile.js b/proxy/build/loadEnvFile.js new file mode 100644 index 00000000000..22d6e43982b --- /dev/null +++ b/proxy/build/loadEnvFile.js @@ -0,0 +1,21 @@ +module.exports = function loadEnvFile() { + const envPath = require('path').join(__dirname, "..", "..", ".env"); + try { + const envContent = fs.readFileSync(envPath, "utf8"); + const envVars = envContent + .split("\n") + .filter((line) => line.trim() && !line.startsWith("#")) + .reduce((vars, line) => { + const [key, ...valueParts] = line.split("="); + const value = valueParts.join("=").trim(); + if (key && value) { + vars[key.trim()] = value.replace(/^["']|["']$/g, ""); + } + return vars; + }, {}); + + Object.assign(process.env, envVars); + } catch (error) { + console.error("Error reading .env file:", error); + } +} \ No newline at end of file diff --git a/proxy/config/auto-ssl.conf b/proxy/config/auto-ssl.conf new file mode 100644 index 00000000000..5b2b622e01d --- /dev/null +++ b/proxy/config/auto-ssl.conf @@ -0,0 +1,19 @@ +# Dynamic handler for issuing or returning certs for SNI domains. +ssl_certificate_by_lua_block { + auto_ssl:ssl_certificate() +} + +# Endpoint used for performing domain verification with Let's Encrypt. +location /.well-known/acme-challenge/ { + content_by_lua_block { + auto_ssl:challenge_server() + } +} + +# You must still define a static ssl_certificate file for nginx to start. +# The SSL certificate we use is generated by https://github.com/kshcherban/acme-nginx +# for the "*.blot.im" and "blot.im" domains. This allows us to do an infinite +# number of subdomains for previewing templates. The paths to these files are +# hardcoded in the source for kshcherban/acme-nginx. I wrote more about this +# process in notes/wildcard-ssl +{{> wildcard-ssl.conf}} diff --git a/proxy/config/blot-blogs.conf b/proxy/config/blot-blogs.conf new file mode 100644 index 00000000000..90de63126c7 --- /dev/null +++ b/proxy/config/blot-blogs.conf @@ -0,0 +1,92 @@ + +set $upstream_server blot_blogs_node; +root /; + +# Match urls which start with /draft/stream/ +location /draft/stream/ { + {{> reverse-proxy-sse.conf}} +} + +# This is used to determine whether the server is handling +# requests, don't remove it unless you change monit.rc too! +location = /health { + return 200; +} + +# This is used to prevent people from accessing the git repositories in user folders +location ^~ /.git { + return 404; +} + +# return 404 immediately for all requests to URLs ending in .php, .asp, .aspx, .jsp, .php5, .php7, .php8 +location ~* \.(?:php[0-9]?|asp|aspx|jsp)$ { + return 404; +} + +# return 404 immediate for /.vscode/sftp.json +location = /.vscode/sftp.json { + return 404; +} + +location = /sftp-config.json { + return 404; +} + +location = /magento_version { + return 404; +} + +# return 404 immediately for all requests to URLs starting with /api/v1 +location ~* ^/api/v1 { + return 404; +} + +location ~* ^/api/collections/ { + return 404; +} + +location ~* ^/vendor/phpunit/ { + return 404; +} + +# return 404 for all URLS starting with /admin/controller/extension/extension +location ~* ^/admin/controller/extension/extension { + return 404; +} + +# return 404 immediately for all requests to URLs starting with /admin/.git/ +location ~* ^/admin/.git { + return 404; +} + +# return 404 immediately for all requests containing 'index.php/' or 'admin.php/' anywhere in their path +location ~* (?:index|admin)\.php/ { + return 404; +} + +# return 404 immediately for all requests to URLs ending in .env +location ~* \.env$ { + return 404; +} + +# return 404 immediately for all requests to URLs containing '/wp-admin/', '/wp-content/', '/wp-includes/' anywhere in their path +location ~* wp-(?:admin|content|includes|diambar|json|config) { + return 404; +} + +# bypass the cache for requests to /random +# todo: handle this with headers instead of a separate location block +location = /random { + {{> reverse-proxy.conf}} +} + +# error page for when node server is unavailable +location = /error-upstream-offline.html { + root {{{config_directory}}}/html; + try_files /error-upstream-offline-blog.html =404; +} + +location / { + {{> reverse-proxy-cache.conf}} +} + diff --git a/proxy/config/blot-site.conf b/proxy/config/blot-site.conf new file mode 100644 index 00000000000..fe55bd0e444 --- /dev/null +++ b/proxy/config/blot-site.conf @@ -0,0 +1,72 @@ +set $upstream_server blot_node; + +# Match /sites/{USERNAME}/status where {USERNAME} is dynamic +location ~ ^/sites/[^/]+/status$ { + {{> reverse-proxy-sse.conf}} +} + +location = /health { + return 200; +} + +# redirect for /cdn/XYZ to cdn.blot.im/XYZ +location ~ ^/cdn/(.*)$ { + return 301 https://cdn.blot.im/$1; +} + +# bypass cache +location = /redis-health { + {{> reverse-proxy.conf}} +} + +# bypass cache +location /sites { + client_max_body_size 100M; + {{> reverse-proxy.conf}} +} + +# bypass cache +location /dashboard { + client_max_body_size 100M; + {{> reverse-proxy.conf}} +} + +# bypass cache +location /questions/ask { + {{> reverse-proxy.conf}} +} + +location ~ ^/questions/[^/]+/edit$ { + {{> reverse-proxy.conf}} +} + +location ~ ^/questions/[^/]+/new$ { + {{> reverse-proxy.conf}} +} + +# git and icloud client need large bodies +location /clients/git { + client_max_body_size 1000M; + {{> reverse-proxy.conf}} +} + +location /clients/icloud { + client_max_body_size {{iCloud_max_body_size}}; + {{> reverse-proxy.conf}} +} + +location /clients { + client_max_body_size 100M; + {{> reverse-proxy.conf}} +} + +# error page for when node server is unavailable +location = /error-upstream-offline.html { + root {{{config_directory}}}/html; + try_files /error-upstream-offline-site.html =404; +} + +location / { + {{> reverse-proxy-cache.conf}} +} + diff --git a/proxy/config/cacher.lua b/proxy/config/cacher.lua new file mode 100644 index 00000000000..aea1da77164 --- /dev/null +++ b/proxy/config/cacher.lua @@ -0,0 +1,384 @@ +local cacher = { + _VERSION = "cacher.lua 1.0.0", + _DESCRIPTION = "Purgable cache for OpenResty", + _URL = "", + _LICENSE = "" + } + +local function cacher_add (self, host, cache_key) + local shared_dictionary = self.shared_dictionary + local cache_key_hash = ngx.md5(cache_key) + ngx.log(ngx.NOTICE, "add hash=" .. cache_key_hash .. " host=" .. host .. " key=" .. cache_key) + shared_dictionary:rpush(host, cache_key_hash) +end + +local function cacher_inspect (self, ngx) + local cache_directory = self.cache_directory + local shared_dictionary = self.shared_dictionary + + if (cache_directory == nil) then + ngx.say("please set cache_directory") + ngx.exit(ngx.OK) + end + + if (shared_dictionary == nil) then + ngx.say("please set shared_dictionary") + ngx.exit(ngx.OK) + end + + -- the host is passed in ?host=example.com + local host = ngx.var.arg_host + + if (host == nil) then + ngx.say("please pass host to inspect as an argument") + ngx.exit(ngx.OK) + end + + ngx.log(ngx.NOTICE, "inspecting host: " .. host) + + local hash_list = {}; + local total_keys = shared_dictionary:llen(host) + + ngx.log(ngx.NOTICE, "found cache keys: " .. total_keys) + + local cache_key_hash = shared_dictionary:lpop(host) + + while cache_key_hash do + table.insert(hash_list, cache_key_hash) + cache_key_hash = shared_dictionary:lpop(host) + end + + -- reinstate the keys in the list + for _, hash in ipairs(hash_list) do + shared_dictionary:rpush(host, hash) + end + + -- append the list of cache keys to the message seperated by newlines + local message = table.concat(hash_list, "\n") + + ngx.say(message) + ngx.exit(ngx.OK) +end + +-- this file will read the contents of the cache directory and store them in +-- the lua shared dict shared_dictionary so we can purge them later + +-- I'm not sure why we need to deduplicate the keys?-- +local function deduplicate_key_list_by_host (host, shared_dict) + + ngx.log(ngx.NOTICE, "deduplicate_key_list_by_host: " .. host) + + local deduplicated_key_list = {} + + -- we lpop from the list until we get nil + local cache_key_hash = shared_dict:lpop(host) + + while cache_key_hash do + + if (deduplicated_key_list[cache_key_hash] == nil) then + table.insert(deduplicated_key_list, cache_key_hash) + deduplicated_key_list[cache_key_hash] = true + else + ngx.log(ngx.NOTICE, "duplicate hash " .. cache_key_hash) + end + + cache_key_hash = shared_dict:lpop(host) + end + + -- reinsert the keys into the list + for _, cache_key_hash in ipairs(deduplicated_key_list) do + ngx.log(ngx.NOTICE, "reinserting " .. cache_key_hash) + shared_dict:rpush(host, cache_key_hash) + end +end + +local function extractHostFromCacheFile (ngx, cache_file_path) + + -- we need to read the first line of the cache file to get the host + -- the first line is in the format: + -- KEY: http://example.com/xyz/abc + local file = io.open(cache_file_path, "r") + local first_line = file:read() + + local number_of_lines_read = 1 + + -- keep reading the file until we get a line that contains "KEY: ", up to a max of 10 lines + while (first_line ~= nil and number_of_lines_read < 10 and string.match(first_line, "KEY: ") == nil) do + first_line = file:read() + end + + if (first_line == nil) then + return nil + end + + local key = string.match(first_line, "KEY: (.*)") + + if (key == nil) then + ngx.log(ngx.NOTICE, "key is nil") + return nil + end + + -- the protocol is included in the uri so we need to remove it + local uri_without_protocol = string.match(key, "://(.*)") + + if (uri_without_protocol == nil) then + return nil + end + + -- the host is the first part of the uri, up to question mark or slash or colon if there is one + local host = string.match(uri_without_protocol, "([^/?#:]+)") + + if (host == nil) then + return nil + end + + file:close() + + return host +end + +-- returns a list of file or directory names in the given directory +local function readdirectory(directory) + local i, t, popen = 0, {}, io.popen + local pfile = popen('ls -a "'..directory..'"') + + -- we want to skip the lines that are . or .. + for filename in pfile:lines() do + if (filename ~= "." and filename ~= "..") then + i = i + 1 + ngx.log(ngx.NOTICE, "found file: " .. filename) + t[i] = filename + end + end + + pfile:close() + return t +end + + +local function cacher_rehydrate (self) + + -- we need to read the contents of the cache directory and store them in + -- the lua shared dict shared_dictionary so we can purge them later + local cache_directory = self.cache_directory + + -- local purged_files = purge_host(ngx.var.arg_host) + local shared_dictionary = self.shared_dictionary + + local message = '' + + ngx.log(ngx.NOTICE, "rehydrate: " .. cache_directory ) + + -- first we list all the top level directories in the cache directory + local top_level_directories = readdirectory(cache_directory) + + -- store a list of hosts + local hosts = {} + + -- then for each directory we list all the files in that directory + for _, top_level_directory in ipairs(top_level_directories) do + + local top_level_directory_path = cache_directory .. "/" .. top_level_directory + local second_level_directories = readdirectory(top_level_directory_path) + + for _, second_level_directory in ipairs(second_level_directories) do + local second_level_directory_path = top_level_directory_path .. "/" .. second_level_directory + local files = readdirectory(second_level_directory_path) + + for _, cache_key_hash in ipairs(files) do + local cache_file_path = top_level_directory .. "/" .. second_level_directory .. "/" .. cache_key_hash + local host = extractHostFromCacheFile(ngx, second_level_directory_path .. "/" .. cache_key_hash) + + + -- if the host was not parsed, log the file + if (host == nil) then + ngx.log(ngx.NOTICE, "rehydrate: failed to parse host: " .. cache_file_path) + message = message .. cache_file_path .. "\n" + else + -- add the host to the list of hosts + if (hosts[host] == nil) then + table.insert(hosts, host) + hosts[host] = true + end + + ngx.log(ngx.NOTICE, "rehydrate: adding FILE=" .. cache_key_hash .. " HOST=" .. host ) + shared_dictionary:rpush(host, cache_key_hash) + end + end + end + end + + for _, host in ipairs(hosts) do + deduplicate_key_list_by_host(host, shared_dictionary) + end + + ngx.log(ngx.NOTICE, "rehydrate: complete") + + if (message == '') then + message = "OK" + end + + return message +end + +local function cacher_purge (self, ngx) + + local cache_directory = self.cache_directory + local shared_dictionary = self.shared_dictionary + + local message = '' + + if (cache_directory == nil) then + ngx.say("please set cache_directory") + ngx.exit(ngx.OK) + end + + if (shared_dictionary == nil) then + ngx.say("please set shared_dictionary") + ngx.exit(ngx.OK) + end + + -- prevent an error if the args are nil + if (ngx.var.args == nil) then + ngx.say("please pass host to purge as an argument") + ngx.exit(ngx.OK) + end + + for host in string.gmatch(ngx.var.args, "host=([^&]+)") do + ngx.log(ngx.NOTICE, "purging host: " .. host) + local total_keys = purge_host(host, shared_dictionary, cache_directory) + message = message .. host .. ": " .. total_keys .. "\n" + end + + -- if message is empty then replace it with a message saying that no hosts were purged + if (message == '') then + message = "no hosts were purged" + end + + ngx.say(message) + ngx.exit(ngx.OK) +end + +function purge_host (host, shared_dictionary, cache_directory) + local total_keys = shared_dictionary:llen(host) + local cache_key_hash = shared_dictionary:lpop(host) + while cache_key_hash do + -- the cache file path is in the following format: $x/$y/$cache_key_hash + -- where x is the last character of the cache_key_hash + -- and y are the two characters before x + local x = cache_key_hash:sub(-1) + local y = cache_key_hash:sub(-3,-2) + local cached_file_path = cache_directory .. "/" .. x .. "/" .. y .. "/" .. cache_key_hash + local f = io.open(cached_file_path, "r") + + if f ~= nil then + io.close(f) + os.remove(cached_file_path) + end + + cache_key_hash = shared_dictionary:lpop(host) + end + + return total_keys +end + +function purge_lru_hosts (self) + -- returns all the keys in the dictionary, the least recently used at the very end of the list + local hosts = self.shared_dictionary:get_keys(0) + local maximum_hosts_to_purge = 500 + + -- we want to create a list of hosts that we can purge from the end of the list of hosts + local number_of_hosts_purged = 0 + + -- we want to purge the least recently used hosts first + for i = #hosts, 1, -1 do + local host = hosts[i] + purge_host(host, self.shared_dictionary, self.cache_directory) + number_of_hosts_purged = number_of_hosts_purged + 1 + if (number_of_hosts_purged >= maximum_hosts_to_purge) then + break + end + end +end + +function cacher_check_free_space (self, ngx) + local minimum_free_space = self.minimum_free_space + + -- if minimum_free_space is not nil then we need to check if we need to purge the lru hosts + if (minimum_free_space ~= nil) then + local free_space = self.shared_dictionary:free_space() + + if (free_space < minimum_free_space) then + ngx.log(ngx.NOTICE, "dictionary_free_space=" .. free_space .. " is less than " .. minimum_free_space .. ", purging lru hosts") + purge_lru_hosts(self) + ngx.log(ngx.NOTICE, "purge of lru hosts complete") + else + ngx.log(ngx.NOTICE, "dictionary_free_space=" .. free_space .. " is greater than " .. minimum_free_space .. ", no need to purge lru hosts") + end + end +end + +function cacher_monitor_free_space (self, ngx, monitor_interval) + + if (monitor_interval == nil) then + monitor_interval = 60 + end + + ngx.timer.every(monitor_interval, function (premature) + if premature then + return + end + + -- if both cache_directory and shared_dictionary are set then we can rehydrate + if (self.shared_dictionary ~= nil) then + local free_space = self.shared_dictionary:free_space() + + local capacity = self.shared_dictionary:capacity() + + -- calculate the memory usage in megabytes, rounded down + local usage = math.floor((capacity - free_space) / 1024 / 1024) + local free_space_mb = math.floor(free_space / 1024 / 1024) + + -- retrieve the disk usage of the cache directory + local cache_directory = self.cache_directory + local handle = io.popen("du -sh " .. cache_directory) + local output = handle:read('*a') + + if (output == nil) then + output = "n/a" + else + output = string.match(output, "([^\t]+)") + end + + handle:close() + ngx.log(ngx.NOTICE, "dictionary_usage=" .. usage .. "M disk_usage=" .. output .. " dictionary_free_space=" .. free_space_mb .. "M") + + cacher_check_free_space(self, ngx) + end + end, self) +end + +--- Create a new cacher instance. +function cacher.new() + + local function cacher_set(self, key, value) + self[key] = value + + -- if both cache_directory and shared_dictionary are set then we can rehydrate + if (self.cache_directory ~= nil and self.shared_dictionary ~= nil) then + cacher_rehydrate(self) + end + end + + return { + purge = cacher_purge, + set = cacher_set, + add = cacher_add, + inspect = cacher_inspect, + rehydrate = cacher_rehydrate, + monitor_free_space = cacher_monitor_free_space, + check_free_space = cacher_check_free_space + } +end + +return cacher \ No newline at end of file diff --git a/proxy/config/http.conf b/proxy/config/http.conf new file mode 100644 index 00000000000..34b740eaf2b --- /dev/null +++ b/proxy/config/http.conf @@ -0,0 +1,70 @@ +log_format access_log_format '[$time_local] $request_id $status $request_time $request_length:$bytes_sent $scheme://$host$request_uri cache=$sent_http_blot_cache'; + +error_log {{{log_directory}}}/error.log info; +access_log {{{log_directory}}}/access.log access_log_format; + +# Hide the nginx version in the server header +server_tokens off; + +# Added to set the content-type charset header +# for text files served by NGINX +charset utf-8; + +{{> static-file.conf}} + +{{#lua_package_path}} + +# Required to resolve the luarocks module 'resty.auto-ssl' +# when running openresty on Github Actions for some reason +lua_package_path '{{{lua_package_path}}};;'; + +{{/lua_package_path}} + +{{^lua_package_path}} +lua_package_path '{{{config_directory}}}/?.lua;;'; +{{/lua_package_path}} + +{{> init.conf}} + +# Make sure this directory exists +# It does not need to be owned by root +# https://serverfault.com/questions/1029358/nginx-keys-zone-size-persistence-and-maximum-number-of-files +# 1m = ~8000 items +# I believe the nginx memory cache does not need to be as large as cacher_dictionary because it can fall back +# to disk in an elegant way +proxy_cache_path {{{cache_directory}}} levels=1:2 keys_zone=PROXY_CACHE:30m inactive=1y max_size=200g use_temp_path=off; + +lua_shared_dict cacher_dictionary 75m; + +upstream blot_node { + # Use least connections algorithm to better distribute load + least_conn; + # Activates the cache for connections to upstream servers. + keepalive 120; + + # We use the default fail_timeout=10s and max_fails=1 since + # there is a backup server which should never be marked as + # unavailable using the max_fails=0 directive + server 127.0.0.1:8089; # blot-container-green MASTER server + server 127.0.0.1:8088 backup max_fails=0; # blot-container-blue FAILOVER server +} + +upstream blot_blogs_node { + # Use least connections algorithm to better distribute load + least_conn; + # Activates the cache for connections to upstream servers. + keepalive 120; + + # We use the default fail_timeout=10s and max_fails=1 since + # there is a backup server which is never marked as unavailable + # We bias traffic to one container slightly so if there is a memory + # leak both containers do not go down close together + server 127.0.0.1:8090 weight=4; # blot-container-yellow + server 127.0.0.1:8091 weight=3; # blot-container-purple + server 127.0.0.1:8088 backup max_fails=0; # blot-container-blue FAILOVER server +} + + +upstream blot_stats { + server 127.0.0.1:19999; +} \ No newline at end of file diff --git a/proxy/config/init.conf b/proxy/config/init.conf new file mode 100644 index 00000000000..2dd81affe63 --- /dev/null +++ b/proxy/config/init.conf @@ -0,0 +1,134 @@ +# The "auto_ssl" shared dict should be defined with enough storage space to +# hold your certificate data. 1MB of storage holds certificates for +# approximately 100 separate domains. Note that this should not cause an +# error if there are too many domains, just that nginx will have to look +# up the certificate in the database which is slower. +# 100mb = 10,000 domains +lua_shared_dict auto_ssl 100m; + +# The "auto_ssl" shared dict is used to temporarily store various settings +# like the secret used by the hook server on port 8999. Do not change or omit. +lua_shared_dict auto_ssl_settings 64k; + +# A DNS resolver must be defined for OCSP stapling to function. +# +# This example uses Google's DNS server. You may want to use your system's +# default DNS servers, which can be found in /etc/resolv.conf. If your network +# is not IPv6 compatible, you may wish to disable IPv6 results by using the +# "ipv6=off" flag (like "resolver 8.8.8.8 ipv6=off"). +# https://github.com/auto-ssl/lua-resty-auto-ssl/issues/12#issuecomment-259402817 +# found by running cat /etc/resolv.conf and looking for the nameserver value +resolver 8.8.8.8 ipv6=off; + +# Internal server running on port 8999 for handling certificate tasks. +server { + + listen 127.0.0.1:8999; + + # Increase the body buffer size, to ensure the internal POSTs can always + # parse the full POST contents into memory. + client_body_buffer_size 128k; + client_max_body_size 128k; + + location / { + content_by_lua_block { + auto_ssl:hook_server() + } + } +} + +init_by_lua_block { + + cacher = (require "cacher").new() + cacher:set("cache_directory", "{{{cache_directory}}}") + cacher:set("shared_dictionary", ngx.shared.cacher_dictionary) + cacher:set("minimum_free_space", 1024 * 1024 * 2) + + auto_ssl = (require "resty.auto-ssl").new() + + local redis = require "resty.redis" + + local redis_options = { host = "{{redis.host}}", port = 6379 , prefix = "ssl" } + + local function get_redis_instance(redis_options) + + local instance = ngx.ctx.auto_ssl_redis_instance + + if instance then + return instance + end + + instance = redis:new() + + local ok, err + + if redis_options["socket"] then + ok, err = instance:connect(redis_options["socket"]) + else + ok, err = instance:connect(redis_options["host"], redis_options["port"]) + end + + if not ok then + return false, err + end + + if redis_options["auth"] then + ok, err = instance:auth(redis_options["auth"]) + if not ok then + return false, err + end + end + + ngx.ctx.auto_ssl_redis_instance = instance + return instance + end + + auto_ssl:set("redis", redis_options) + + -- Certificates are stored in redis + auto_ssl:set("storage_adapter", "resty.auto-ssl.storage_adapters.redis") + + -- This function determines whether the incoming domain + -- should automatically issue a new SSL certificate. + -- I need to set domain:blot.im to foo in the database so that + -- the allow_domain function works as expected even though + -- it's not technically a user's domain + + local function allow_domain(domain) + + local certstorage = auto_ssl.storage + + local fullchain_pem, privkey_pem = certstorage:get_cert(domain) + + -- If we have this cert in the memory cache + -- then return it without checking redis to save time + if fullchain_pem then + return true + end + + local redis_instance, instance_err = get_redis_instance(redis_options) + + if instance_err then + return nil, instance_err + end + + local res, err = redis_instance:get('domain:' .. domain) + + if res == ngx.null then + return false + end + + return true + end + + auto_ssl:set("allow_domain", allow_domain) + + auto_ssl:init() +} + +init_worker_by_lua_block { + auto_ssl:init_worker() + if ngx.worker.id() == 0 then + cacher:monitor_free_space(ngx) + end +} \ No newline at end of file diff --git a/proxy/config/initial.conf b/proxy/config/initial.conf new file mode 100644 index 00000000000..82d375b21b8 --- /dev/null +++ b/proxy/config/initial.conf @@ -0,0 +1,16 @@ +user {{user}} {{user}}; ## Default: nobody + +worker_processes 2; + +# Defines the scheduling priority for worker processes like it is done by the nice command: +# a negative number means higher priority. Allowed range normally varies from -20 to 20. +worker_priority -20; + +# Sets the limit of the maximum number of open files (RLIMIT_NOFILE) +# for worker processes +worker_rlimit_nofile 10000; + +events { + worker_connections 10000; + multi_accept on; +} \ No newline at end of file diff --git a/proxy/config/reverse-proxy-cache.conf b/proxy/config/reverse-proxy-cache.conf new file mode 100644 index 00000000000..b1a7c86d401 --- /dev/null +++ b/proxy/config/reverse-proxy-cache.conf @@ -0,0 +1,26 @@ +set $cache_key $scheme://$host$request_uri; + +open_file_cache off; + +proxy_cache PROXY_CACHE; + +# https://cassiomolin.com/2016/09/09/which-http-status-codes-are-cacheable/ +proxy_cache_valid 200 301 302 400 404 1y; + +# we want to cache everything, even if cache-control says not to +# it's important to ignore the vary header from upstream otherwise +# nginx will cache multiple copies of the same page +proxy_ignore_headers Cache-Control Expires Set-Cookie Vary; + +proxy_cache_key $cache_key; +proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504; + +{{> reverse-proxy.conf}} + +# This registers a cached response in our dictionary of cached responses +# pass in the proxy_cache_key setting +log_by_lua_block { + if ngx.var.upstream_cache_status == "MISS" then + cacher:add(ngx.var.host, ngx.var.cache_key) + end +} \ No newline at end of file diff --git a/proxy/config/reverse-proxy-sse.conf b/proxy/config/reverse-proxy-sse.conf new file mode 100644 index 00000000000..7a39cc8bed4 --- /dev/null +++ b/proxy/config/reverse-proxy-sse.conf @@ -0,0 +1,17 @@ +proxy_pass http://blot_node; +proxy_read_timeout 24h; +proxy_send_timeout 24h; +proxy_connect_timeout 24h; + +# SSE-specific configurations +proxy_http_version 1.1; # SSE requires HTTP/1.1 +proxy_set_header Connection ''; # Disable keep-alive +proxy_buffering off; # Disable response buffering +proxy_cache off; # Ensure no caching + +# generic +proxy_set_header Host $host; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $scheme; +proxy_set_header X-Request-ID $request_id; diff --git a/proxy/config/reverse-proxy.conf b/proxy/config/reverse-proxy.conf new file mode 100644 index 00000000000..9648cc40a57 --- /dev/null +++ b/proxy/config/reverse-proxy.conf @@ -0,0 +1,48 @@ +add_header 'Blot-Server' '{{server_label}}' always; +add_header 'Blot-Cache' $upstream_cache_status always; +add_header 'Blot-Upstream' $upstream_addr always; + +proxy_pass http://$upstream_server; +proxy_http_version 1.1; + +proxy_set_header Host $host; +proxy_set_header X-Real-IP $remote_addr; +proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +proxy_set_header X-Forwarded-Proto $scheme; +proxy_set_header X-Request-ID $request_id; +proxy_set_header Upgrade $http_upgrade; +proxy_set_header Connection $http_connection; # Use $http_connection for better dynamic handling +# We let nginx handle the compression +proxy_set_header Accept-Encoding ""; + +gzip on; +gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript; +gzip_comp_level 6; +gzip_proxied any; + +# For larger responses (blog posts, content-heavy pages) +# Configured for 2MB total buffer size +proxy_buffers 32 64k; # 2MB total (32 * 64k = 2048k) +proxy_buffer_size 64k; + +proxy_redirect off; + +proxy_intercept_errors on; +error_page 500 502 503 504 429 /error-upstream-offline.html; + +# Enable retrying the next upstream on failure +# Important to limit the number of tries and possibly +# the timeout otherwise a single request which triggers an error could take down all the servers +# We do rolling deploys where one server is taken down at a time so this should allow +# a failover in that case. Expensive or bug-triggering requests should consistently failwhale +# You have to be careful that these error codes are only returned when the server is truly +# unavailable – I ran into a nasty bug where a request that should have lead to a 404 was return +# a 500 error code, and this config considered the upstream server bad! +proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; +proxy_next_upstream_tries 1; +proxy_next_upstream_timeout 10s; + +# Fine-tune timeouts for better failover +proxy_connect_timeout 3s; # Time to establish a connection to the upstream +proxy_send_timeout 10s; # Time to send the request to the upstream +proxy_read_timeout 15s; # Time to read the response from the upstream diff --git a/proxy/config/server.conf b/proxy/config/server.conf new file mode 100644 index 00000000000..3378895d8f2 --- /dev/null +++ b/proxy/config/server.conf @@ -0,0 +1,215 @@ +{{> initial.conf}} + +http { + + {{> http.conf}} + + # blot subdomains (e.g. david.blot.im) + server { + listen 80; + listen 443 ssl; + + {{^disable_http2}} + http2 on; + {{/disable_http2}} + + # match all subdomains of blot.im which do not start with preview- + # e.g. blog-on-david.blot.im + server_name "~^(?!preview-)[^.]+\.{{host}}$"; + + {{> wildcard-ssl.conf}} + + {{> blot-blogs.conf}} + } + + # preview subdomains (e.g. preview-of-blog-on-david.blot.im) + # these skip the cache and are passed directly to node + server { + listen 80; + listen 443 ssl; + + {{^disable_http2}} + http2 on; + {{/disable_http2}} + + server_name "~^preview-[^.]+\.{{host}}$"; + + {{> wildcard-ssl.conf}} + + location / { + set $upstream_server blot_blogs_node; + {{> reverse-proxy.conf}} + } + } + + # cdn subdomain (e.g. cdn.blot.im) + # these skip the cache and are passed directly to node + server { + listen 80; + listen 443 ssl; + + {{^disable_http2}} + http2 on; + {{/disable_http2}} + + server_name cdn.{{host}}; + + {{> wildcard-ssl.conf}} + + location / { + set $upstream_server blot_blogs_node; + {{> reverse-proxy.conf}} + } + } + + # stats subdomain + # these skip the cache and are passed directly to node + server { + listen 80; + listen 443 ssl; + {{^disable_http2}} + http2 on; + {{/disable_http2}} + + server_name stats.{{host}}; + + {{> wildcard-ssl.conf}} + + auth_basic "Protected"; + auth_basic_user_file /home/ec2-user/netdataconfig/netdata/passwords; + + # Redirect root to /v3 to avoid annoying login window + location = / { + return 301 $scheme://$host/v3/; + } + + location / { + proxy_pass http://localhost:19999; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Authorization $http_authorization; + + # WebSocket support + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Timeouts + proxy_connect_timeout 60s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + } + } + + + # blot.im + server { + listen 443 ssl; + + {{^disable_http2}} + http2 on; + {{/disable_http2}} + + server_name {{host}}; + + {{> wildcard-ssl.conf}} + + {{> blot-site.conf}} + } + + # webhooks relay at webhooks.blot.im + server { + listen 443 ssl; + {{^disable_http2}} + http2 on; + {{/disable_http2}} + server_name webhooks.{{host}}; + + {{> auto-ssl.conf}} + + location / { + # the icloud relay needs to handle large file uploads + client_max_body_size {{webhooks_client_max_body_size}}; + + # IMPORTANT – we only send webhook relay requests to one + # upstream (the master node - blot-container-green) because + # it has an inmemory dictionary of connected clients – if + # you have multiple upstreams, it doesn't work – + # DO NOT replace with http://blot_node + proxy_pass http://127.0.0.1:8089; + + proxy_read_timeout 24h; + proxy_send_timeout 24h; + proxy_connect_timeout 24h; + + # SSE-specific configurations + proxy_http_version 1.1; # SSE requires HTTP/1.1 + proxy_set_header Connection ''; # Disable keep-alive + proxy_buffering off; # Disable response buffering + proxy_cache off; # Ensure no caching + + # generic + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Request-ID $request_id; + } + } + + # custom domains + server { + listen 80 default_server; + listen 443 ssl default_server; + {{^disable_http2}} + http2 on; + {{/disable_http2}} + {{> auto-ssl.conf}} + {{> blot-blogs.conf}} + } + + # redirect blot.im over HTTP to HTTPS + server { + listen 80; + server_name {{host}}; + return 301 https://$host$request_uri; + } + + # internal server for inspecting and purging the cache + server { + listen 127.0.0.1:80; + {{#openresty_instance_private_ip}} + # needed by the node application running inside docker + # which doesn't have access to the host network + # this seems to cause errors for custom domains + # and overrides the default server + listen {{openresty_instance_private_ip}}:8077; + {{/openresty_instance_private_ip}} + + location = /inspect { + content_by_lua_block { + cacher:inspect(ngx) + } + } + + location = /rehydrate { + content_by_lua_block { + local message = cacher:rehydrate(ngx) + ngx.say(message) + } + } + + location = /purge { + content_by_lua_block { + cacher:purge(ngx) + } + } + + # otherwise, return 404 + location / { + return 404; + } + } +} \ No newline at end of file diff --git a/proxy/config/static-file.conf b/proxy/config/static-file.conf new file mode 100644 index 00000000000..d5303f6e507 --- /dev/null +++ b/proxy/config/static-file.conf @@ -0,0 +1,163 @@ +# Taken from +# https://github.com/h5bp/server-configs-nginx/blob/master/mime.types + +types { + + # Data interchange + + application/atom+xml atom; + application/json json map topojson; + application/ld+json jsonld; + application/rss+xml rss; + application/vnd.geo+json geojson; + application/xml rdf xml; + + + # JavaScript + + # Normalize to standard type. + # https://tools.ietf.org/html/rfc4329#section-7.2 + application/javascript js jsgzip; + + + # Manifest files + + application/manifest+json webmanifest; + application/x-web-app-manifest+json webapp; + text/cache-manifest appcache; + + + # Media files + + audio/midi mid midi kar; + audio/mp4 aac f4a f4b m4a; + audio/mpeg mp3; + audio/ogg oga ogg opus; + audio/x-realaudio ra; + audio/x-wav wav; + image/bmp bmp; + image/gif gif; + image/jpeg jpeg jpg; + image/jxr jxr hdp wdp; + image/png png; + image/svg+xml svg svggzip; + image/tiff tif tiff; + image/vnd.wap.wbmp wbmp; + image/webp webp; + image/x-jng jng; + video/3gpp 3gp 3gpp; + video/mp4 f4p f4v m4v mp4; + video/mpeg mpeg mpg; + video/ogg ogv; + video/quicktime mov; + video/webm webm; + video/x-flv flv; + video/x-mng mng; + video/x-ms-asf asf asx; + video/x-ms-wmv wmv; + video/x-msvideo avi; + + # Serving `.ico` image files with a different media type + # prevents Internet Explorer from displaying then as images: + # https://github.com/h5bp/html5-boilerplate/commit/37b5fec090d00f38de64b591bcddcb205aadf8ee + + image/x-icon cur ico; + + + # Microsoft Office + + application/msword doc; + application/vnd.ms-excel xls; + application/vnd.ms-powerpoint ppt; + application/vnd.openxmlformats-officedocument.wordprocessingml.document docx; + application/vnd.openxmlformats-officedocument.spreadsheetml.sheet xlsx; + application/vnd.openxmlformats-officedocument.presentationml.presentation pptx; + + + # Web fonts + + application/font-woff woff; + application/font-woff2 woff2; + application/vnd.ms-fontobject eot; + + # Browsers usually ignore the font media types and simply sniff + # the bytes to figure out the font type. + # https://mimesniff.spec.whatwg.org/#matching-a-font-type-pattern + # + # However, Blink and WebKit based browsers will show a warning + # in the console if the following font types are served with any + # other media types. + + application/x-font-ttf ttc ttf; + font/opentype otf; + + + # Other + + application/java-archive ear jar war; + application/mac-binhex40 hqx; + application/octet-stream bin deb dll dmg exe img iso msi msm msp safariextz; + application/pdf pdf; + application/postscript ai eps ps; + application/rtf rtf; + application/vnd.google-earth.kml+xml kml; + application/vnd.google-earth.kmz kmz; + application/vnd.wap.wmlc wmlc; + application/x-7z-compressed 7z; + application/x-bb-appworld bbaw; + application/x-bittorrent torrent; + application/x-chrome-extension crx; + application/x-cocoa cco; + application/x-java-archive-diff jardiff; + application/x-java-jnlp-file jnlp; + application/x-makeself run; + application/x-opera-extension oex; + application/x-perl pl pm; + application/x-pilot pdb prc; + application/x-rar-compressed rar; + application/x-redhat-package-manager rpm; + application/x-sea sea; + application/x-shockwave-flash swf; + application/x-stuffit sit; + application/x-tcl tcl tk; + application/x-x509-ca-cert crt der pem; + application/x-xpinstall xpi; + application/xhtml+xml xhtml; + application/xslt+xml xsl; + application/zip zip; + text/css css cssgzip; + text/csv csv; + text/html htm html shtml htmlgzip; + text/markdown md; + text/mathml mml; + text/plain txt; + text/vcard vcard vcf; + text/vnd.rim.location.xloc xloc; + text/vnd.sun.j2me.app-descriptor jad; + text/vnd.wap.wml wml; + text/vtt vtt; + text/x-component htc; + +} + +open_file_cache max=1000 inactive=3600s; +open_file_cache_valid 60s; +open_file_cache_min_uses 1; +open_file_cache_errors on; + +# do we get better performance if we disable this feature +# rather than resending the entire file? +# if_modified_since off; + +sendfile on; +sendfile_max_chunk 1m; + +tcp_nopush on; +tcp_nodelay on; + +gzip on; +gzip_types text/plain text/xml text/css + text/comma-separated-values + application/javascript + text/javascript application/x-javascript + application/atom+xml; \ No newline at end of file diff --git a/proxy/config/wildcard-ssl.conf b/proxy/config/wildcard-ssl.conf new file mode 100644 index 00000000000..1215aa57e0f --- /dev/null +++ b/proxy/config/wildcard-ssl.conf @@ -0,0 +1,23 @@ +# Check with the SSL evaluator tool +ssl_protocols TLSv1.2 TLSv1.3; +ssl_prefer_server_ciphers off; +ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384"; + +# Hopefully improves performance +# one megabyte can store about 4000 sessions +# 10m = 40,000 sessions +ssl_session_cache shared:SSL:10m; +ssl_session_timeout 1h; +ssl_session_tickets off; + +# Add OCSP stapling if your cert supports it +ssl_stapling on; +ssl_stapling_verify on; + +# The SSL certificate we use is generated by https://github.com/kshcherban/acme-nginx +# for the "*.blot.im" and "blot.im" domains. This allows us to do an infinite +# number of subdomains for previewing templates. The paths to these files are +# hardcoded in the source for kshcherban/acme-nginx. I wrote more about this +# process in notes/wildcard-ssl +ssl_certificate {{{ssl_certificate}}}; +ssl_certificate_key {{{ssl_certificate_key}}}; \ No newline at end of file diff --git a/proxy/entrypoint.sh b/proxy/entrypoint.sh new file mode 100644 index 00000000000..805229ac8eb --- /dev/null +++ b/proxy/entrypoint.sh @@ -0,0 +1,48 @@ +#!/bin/bash +set -e + +CERT_PATH="/etc/ssl/private/letsencrypt-domain.pem" +KEY_PATH="/etc/ssl/private/letsencrypt-domain.key" + +# Enable staging mode if USE_STAGING is set to "true" +STAGING_FLAG="" +if [[ "$USE_STAGING" == "true" ]]; then + echo "Using Let's Encrypt staging environment" + STAGING_FLAG="--staging" +fi + +# Ensure BLOT_HOST is set +if [[ -z "$BLOT_HOST" ]]; then + echo "Error: BLOT_HOST is not set. Please provide the domain via the BLOT_HOST environment variable." >&2 + exit 1 +fi + +echo "BLOT_HOST=$BLOT_HOST" + +# Check if AWS credentials are set +if [[ -n "$AWS_ACCESS_KEY_ID" && -n "$AWS_SECRET_ACCESS_KEY" && -n "$AWS_REGION" ]]; then + echo "AWS credentials detected. Using DNS challenge for wildcard certificate." + DNS_PROVIDER="--dns-provider route53" + DOMAIN="-d *.$BLOT_HOST" +else + echo "AWS credentials not found. Falling back to HTTP challenge for single-domain certificate." + DNS_PROVIDER="" + DOMAIN="-d $BLOT_HOST" +fi + +# Generate SSL certificate if not present +if [[ ! -f "$CERT_PATH" || ! -f "$KEY_PATH" ]]; then + echo "Generating SSL certificate for $DOMAIN..." + acme-nginx \ + $DNS_PROVIDER \ + $DOMAIN \ + -o "$CERT_PATH" \ + --domain-private-key "$KEY_PATH" \ + $STAGING_FLAG + echo "Certificate generated successfully." +else + echo "Certificate already exists. Skipping generation." +fi + +# Start OpenResty +exec /usr/local/openresty/bin/openresty -g "daemon off;" diff --git a/proxy/html/error-upstream-offline-blog.html b/proxy/html/error-upstream-offline-blog.html new file mode 100644 index 00000000000..c346f616f47 --- /dev/null +++ b/proxy/html/error-upstream-offline-blog.html @@ -0,0 +1,81 @@ +Error + + +

This site is offline

+

Our server failed to respond to your request. We have been notified and will fix the problem as soon as possible.

+

+ Contact us + Status page
+

+ + \ No newline at end of file diff --git a/proxy/html/error-upstream-offline-site.html b/proxy/html/error-upstream-offline-site.html new file mode 100644 index 00000000000..b7aefdda3ba --- /dev/null +++ b/proxy/html/error-upstream-offline-site.html @@ -0,0 +1,81 @@ +Error + + +

Blot is offline

+

Our server failed to respond to your request. We have been notified and will fix the problem as soon as possible.

+

+ Contact us + Status page
+

+ + \ No newline at end of file diff --git a/proxy/tests/basic.conf b/proxy/tests/basic.conf new file mode 100644 index 00000000000..258ccceeb71 --- /dev/null +++ b/proxy/tests/basic.conf @@ -0,0 +1,89 @@ +user {{user}} {{group}}; + +events { + worker_connections 1000; + multi_accept on; +} + +http { + + log_format access_log_format '[$time_local] ID=$request_id STATUS=$status REQ_TIME=$request_time REQ_LEN=$request_length RES_LEN=$bytes_sent $scheme://$host$request_uri CACHE_STATUS=$sent_http_cache_status'; + + error_log {{{data_directory}}}/error.log debug; + access_log {{{data_directory}}}/access.log access_log_format; + + proxy_cache_path {{{cache_directory}}} levels=1:2 keys_zone=PROXY_CACHE:10m inactive=1y max_size=10g use_temp_path=off; + + lua_shared_dict cacher_dictionary 50m; + lua_package_path "{{{lua_directory}}}/?.lua;;"; + + init_by_lua_block { + cacher = (require "cacher").new() + cacher:set("cache_directory", "{{{cache_directory}}}") + cacher:set("shared_dictionary", ngx.shared.cacher_dictionary) + } + + server { + listen {{openresty.port}} default_server; + + location = /purge { + content_by_lua_block { + cacher:purge(ngx) + } + } + + location = /health { + return 200 'Hello, World!'; + } + + location / { + set $cache_key $scheme://$host:$server_port$request_uri; + + open_file_cache off; + + proxy_cache PROXY_CACHE; + + # https://cassiomolin.com/2016/09/09/which-http-status-codes-are-cacheable/ + proxy_cache_valid 200 301 302 400 404 1y; + + # we want to cache everything + proxy_ignore_headers Cache-Control Expires Set-Cookie; + + proxy_cache_key $cache_key; + proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504; + + add_header 'Cache-Status' $upstream_cache_status always; + add_header 'Blot-Server' '...'; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Request-ID $request_id; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + + # This seemed to fix an issue with MP4 demo video + # ERR_CONTENT_DECODING_FAILED + # All gzipping should be handled by NGINX, rather than node + proxy_set_header Accept-Encoding ""; + + gzip on; + gzip_comp_level 6; + gzip_proxied any; + + proxy_redirect off; + + proxy_pass http://127.0.0.1:{{node.port}}; + proxy_http_version 1.1; + + # This registers a cached response in our dictionary of cached responses + # pass in the proxy_cache_key setting + log_by_lua_block { + if ngx.var.upstream_cache_status == "MISS" then + cacher:add(ngx.var.host, ngx.var.cache_key) + end + } + } + } +} \ No newline at end of file diff --git a/proxy/tests/basic.js b/proxy/tests/basic.js new file mode 100644 index 00000000000..4c8fc140882 --- /dev/null +++ b/proxy/tests/basic.js @@ -0,0 +1,106 @@ +const fs = require("fs-extra"); +const fetch = require("node-fetch"); +const setup = require("./util/setup"); + +describe("cacher", function () { + setup("./basic.conf"); + + it("caches a request", async function () { + const response = await fetch(this.origin); + const text = await response.text(); + expect(response.status).toBe(200); + expect(response.headers.get("Cache-Status")).toBe("MISS"); + expect(text).toBe("Hello Node!"); + + const files = await this.listCache(); + + expect(files.length).toBe(1); + + const contents = await fs.readFile(files[0], "utf-8"); + // we trim because the lines end with '\r' + const lines = contents.split("\n").map(l => l.trim()); + + expect(lines.length).toBe(12); + expect(lines[1]).toBe("KEY: " + this.origin + "/"); + expect(lines[2]).toBe("HTTP/1.1 200 OK"); + expect(lines[3]).toBe("X-Powered-By: Express"); + expect(lines[4]).toBe("Content-Type: text/html; charset=utf-8"); + expect(lines[5]).toBe("Content-Length: 11"); + expect(lines[6]).toMatch(/^ETag: /); + expect(lines[7]).toMatch(/^Date: /); + expect(lines[8]).toBe("Connection: keep-alive"); + expect(lines[9]).toBe("Keep-Alive: timeout=5"); + + expect(lines[11]).toBe("Hello Node!"); + + // if we retry the request, the header 'cache-hit' should be set + const cachedResponse = await fetch(this.origin); + const cachedText = await cachedResponse.text(); + expect(cachedResponse.status).toBe(200); + expect(cachedResponse.headers.get("Cache-Status")).toBe("HIT"); + expect(cachedText).toBe(text); + }); + + it("purges a cached request", async function () { + const response = await fetch(this.origin); + const text = await response.text(); + expect(response.status).toBe(200); + expect(response.headers.get("Cache-Status")).toBe("MISS"); + const cachedResponse = await fetch(this.origin); + const cachedText = await cachedResponse.text(); + expect(cachedResponse.status).toBe(200); + expect(cachedResponse.headers.get("Cache-Status")).toBe("HIT"); + expect(cachedText).toBe(text); + // use a request to origin/purge with method PURGE and query host '127.0.0.1' to purge the cache + await fetch(this.origin + "/purge?host=127.0.0.1"); + const purgedResponse = await fetch(this.origin); + const purgedText = await purgedResponse.text(); + expect(purgedResponse.status).toBe(200); + expect(purgedResponse.headers.get("Cache-Status")).toBe("MISS"); + expect(purgedText).toBe(text); + }); + + it("purges a cached request even after restarting openresty", async function () { + const response = await fetch(this.origin); + expect(response.status).toBe(200); + expect(response.headers.get("Cache-Status")).toBe("MISS"); + + await this.restartOpenresty(); + + expect((await this.listCache()).length).toEqual(1); + + const purgeResponse = await fetch(this.origin + "/purge?host=127.0.0.1"); + const purgeText = await purgeResponse.text(); + expect(purgeResponse.status).toBe(200); + expect(purgeText.trim()).toBe("127.0.0.1: 1"); + + expect(await this.listCache({ watch: false })).toEqual([]); + }); + + it("can purge multiple hosts", async function () { + const purgeResponse = await fetch(this.origin + "/purge?host=example1.com&host=example2.com"); + const purgeText = await purgeResponse.text(); + + expect(purgeResponse.status).toBe(200); + expect(purgeText.trim()).toBe("example1.com: 0\nexample2.com: 0"); + }); + + it("purges a large number of cached requests", async function () { + const number_of_requests = 500; + + for (let i = 0; i < number_of_requests; i++) { + const res = await fetch(this.origin + "/timestamp/" + i); + const text = await res.text(); + expect(res.status).toBe(200); + expect(parseInt(text)).toBeGreaterThan(0); + expect(res.headers.get("Cache-Status")).toBe("MISS"); + } + + expect((await this.listCache()).length).toEqual(number_of_requests); + + const purgeResponse = await fetch(this.origin + "/purge?host=127.0.0.1"); + const purgeText = await purgeResponse.text(); + expect(purgeResponse.status).toBe(200); + expect(purgeText.trim()).toBe("127.0.0.1: " + number_of_requests); + }); +}); diff --git a/proxy/tests/gzip.conf b/proxy/tests/gzip.conf new file mode 100644 index 00000000000..5bb278f068a --- /dev/null +++ b/proxy/tests/gzip.conf @@ -0,0 +1,72 @@ +user {{user}} {{group}}; + +events { + worker_connections 1000; +} + +http { + + log_format access_log_format '[$time_local] ID=$request_id STATUS=$status REQ_TIME=$request_time REQ_LEN=$request_length RES_LEN=$bytes_sent $scheme://$host$request_uri CACHE_STATUS=$sent_http_cache_status'; + + error_log {{{data_directory}}}/error.log debug; + access_log {{{data_directory}}}/access.log access_log_format; + + proxy_cache_path {{{cache_directory}}} levels=1:2 keys_zone=PROXY_CACHE:10m inactive=1y max_size=10g use_temp_path=off; + + lua_shared_dict cacher_dictionary 50m; + lua_package_path "{{{lua_directory}}}/?.lua;;"; + + init_by_lua_block { + cacher = (require "cacher").new() + cacher:set("cache_directory", "{{{cache_directory}}}") + cacher:set("shared_dictionary", ngx.shared.cacher_dictionary) + } + + server { + listen {{openresty.port}} default_server; + + location = /purge { + content_by_lua_block { + cacher:purge(ngx) + } + } + + location = /health { + return 200 'Hello, World!'; + } + + location / { + + gzip on; + gzip_min_length 1000; + gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript; + gzip_comp_level 9; + gzip_proxied any; + + set $cache_key $scheme://$host:$server_port$request_uri; + open_file_cache off; + + proxy_cache PROXY_CACHE; + + proxy_cache_valid 200 301 302 400 404 1y; + + proxy_ignore_headers Cache-Control Expires Set-Cookie Vary; + + proxy_cache_key $cache_key; + add_header 'Cache-Status' $upstream_cache_status always; + + proxy_redirect off; + + proxy_pass http://127.0.0.1:{{node.port}}; + proxy_http_version 1.1; + + # This registers a cached response in our dictionary of cached responses + # pass in the proxy_cache_key setting + log_by_lua_block { + if ngx.var.upstream_cache_status == "MISS" then + cacher:add(ngx.var.host, ngx.var.cache_key) + end + } + } + } +} \ No newline at end of file diff --git a/proxy/tests/gzip.js b/proxy/tests/gzip.js new file mode 100644 index 00000000000..c2046a72f66 --- /dev/null +++ b/proxy/tests/gzip.js @@ -0,0 +1,55 @@ +const fs = require("fs-extra"); +const fetch = require("node-fetch"); +const setup = require("./util/setup"); +const zlib = require("zlib"); + +describe("cacher handles gzip", function () { + setup("./gzip.conf"); + + // I think this test reproduces a bug on our server where we have multiple files + // seemingly with the same cache key but different md5 hash filenames + // https://github.com/openresty/srcache-nginx-module/issues/11 + // nginx claims it will use the md5 hash of the key to determine the filename + // but it seems that if there is a gzipped and non-gzipped version of the file + // it will produce a different md5 hash + it("does not store multiple cache files when the accept-encoded header is different", async function () { + const response = await fetch(this.origin + "/gzip", { + headers: { "Accept-Encoding": "" } + }); + + expect(response.status).toBe(200); + expect(response.headers.get("Content-Encoding")).toBe(null); + expect(response.headers.get("Cache-Status")).toBe("MISS"); + + const text = await response.text(); + + expect((await this.listCache()).length).toEqual(1); + + const compressedResponse = await fetch(this.origin + "/gzip", { + headers: { "Accept-Encoding": "gzip" } + }); + expect(compressedResponse.status).toBe(200); + expect(compressedResponse.headers.get("Content-Encoding")).toBe("gzip"); + expect(compressedResponse.headers.get("Cache-Status")).toBe("HIT"); + expect(await compressedResponse.text()).toBe(text); + + expect((await this.listCache()).length).toEqual(1); + + // try with the accept-encoding header set to internet explorer 6 + const ie6Response = await fetch(this.origin + "/gzip", { + headers: { "Accept-Encoding": "gzip, deflate" } + }); + expect(ie6Response.status).toBe(200); + expect(ie6Response.headers.get("Cache-Status")).toBe("HIT"); + expect(await ie6Response.text()).toBe(text); + + expect((await this.listCache()).length).toEqual(1); + + const purgeResponse = await fetch(this.origin + "/purge?host=127.0.0.1"); + const purgeText = await purgeResponse.text(); + expect(purgeResponse.status).toBe(200); + expect(purgeText.trim()).toBe("127.0.0.1: 1"); + + expect(await this.listCache({ watch: false })).toEqual([]); + }); +}); diff --git a/proxy/tests/inspect.conf b/proxy/tests/inspect.conf new file mode 100644 index 00000000000..eea961f0c43 --- /dev/null +++ b/proxy/tests/inspect.conf @@ -0,0 +1,73 @@ +user {{user}} {{group}}; + +events { + worker_connections 1000; + multi_accept on; +} + +http { + + log_format access_log_format '[$time_local] ID=$request_id STATUS=$status REQ_TIME=$request_time REQ_LEN=$request_length RES_LEN=$bytes_sent $scheme://$host$request_uri CACHE_STATUS=$sent_http_cache_status'; + + error_log {{{data_directory}}}/error.log debug; + access_log {{{data_directory}}}/access.log access_log_format; + + proxy_cache_path {{{cache_directory}}} levels=1:2 keys_zone=PROXY_CACHE:10m inactive=1y max_size=10g use_temp_path=off; + + lua_shared_dict cacher_dictionary 300k; + lua_package_path "{{{lua_directory}}}/?.lua;;"; + + init_by_lua_block { + cacher = (require "cacher").new() + cacher:set("cache_directory", "{{{cache_directory}}}") + cacher:set("shared_dictionary", ngx.shared.cacher_dictionary) + } + + init_worker_by_lua_block { + cacher:monitor_free_space(ngx, interval=1) + } + + server { + listen {{openresty.port}} default_server; + + location = /purge { + content_by_lua_block { + cacher:purge(ngx) + } + } + + location = /inspect { + content_by_lua_block { + cacher:inspect(ngx) + } + } + + location = /health { + return 200 'Hello, World!'; + } + + location / { + open_file_cache off; + + set $cache_key $scheme://$host:$server_port$request_uri; + + proxy_cache PROXY_CACHE; + proxy_cache_valid 200 301 302 304 1y; + proxy_cache_key $cache_key; + proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504; + + add_header 'Cache-Status' $upstream_cache_status always; + + proxy_pass http://127.0.0.1:{{node.port}}; + proxy_http_version 1.1; + + # This registers a cached response in our dictionary of cached responses + # pass in the proxy_cache_key setting + log_by_lua_block { + if ngx.var.upstream_cache_status == "MISS" then + cacher:add(ngx.var.host, ngx.var.cache_key, ngx.var) + end + } + } + } +} \ No newline at end of file diff --git a/proxy/tests/inspect.js b/proxy/tests/inspect.js new file mode 100644 index 00000000000..a5da493ee89 --- /dev/null +++ b/proxy/tests/inspect.js @@ -0,0 +1,38 @@ +const fs = require("fs-extra"); +const fetch = require("node-fetch"); +const setup = require("./util/setup"); + +describe("cacher", function () { + setup("./inspect.conf"); + + it( + "inspects stored requests", + async function () { + const number_of_requests = 100; + + for (let i = 0; i < number_of_requests; i++) { + const res = await fetch(this.origin + "/timestamp/" + i); + const text = await res.text(); + expect(res.status).toBe(200); + expect(parseInt(text)).toBeGreaterThan(0); + expect(res.headers.get("Cache-Status")).toBe("MISS"); + } + + const files = await this.listCache(); + + expect(files.length).toBe(number_of_requests); + + const inspectResponse = await fetch( + this.origin + "/inspect?host=127.0.0.1" + ); + const inspectText = await inspectResponse.text(); + + expect(inspectText.trim().split("\n").sort()).toEqual( + files.map(path => path.split("/").pop()).sort() + ); + + expect(await this.inspectCache()).toEqual("Cache is consistent"); + }, + 1000 * 60 * 5 + ); +}); diff --git a/proxy/tests/lru_purge.conf b/proxy/tests/lru_purge.conf new file mode 100644 index 00000000000..5e9a76b3c06 --- /dev/null +++ b/proxy/tests/lru_purge.conf @@ -0,0 +1,65 @@ +user {{user}} {{group}}; + +events { + worker_connections 1000; + multi_accept on; +} + +http { + + log_format access_log_format '[$time_local] ID=$request_id STATUS=$status REQ_TIME=$request_time REQ_LEN=$request_length RES_LEN=$bytes_sent $scheme://$host$request_uri CACHE_STATUS=$sent_http_cache_status'; + + error_log {{{data_directory}}}/error.log debug; + access_log {{{data_directory}}}/access.log access_log_format; + + proxy_cache_path {{{cache_directory}}} levels=1:2 keys_zone=PROXY_CACHE:10m inactive=1y max_size=10g use_temp_path=off; + + lua_shared_dict cacher_dictionary 1m; + lua_package_path "{{{lua_directory}}}/?.lua;;"; + + init_by_lua_block { + cacher = (require "cacher").new() + cacher:set("cache_directory", "{{{cache_directory}}}") + cacher:set("shared_dictionary", ngx.shared.cacher_dictionary) + -- purge cache dictionary when space is less than 500kb + cacher:set("minimum_free_space", 1024 * 700) + } + + server { + listen {{openresty.port}} default_server; + + location = /inspect { + content_by_lua_block { + cacher:inspect(ngx) + } + } + + location = /health { + return 200 'Hello, World!'; + } + + location / { + open_file_cache off; + + set $cache_key $scheme://$arg_host:$server_port$request_uri; + + proxy_cache PROXY_CACHE; + proxy_cache_valid 200 301 302 304 1y; + proxy_cache_key $cache_key; + proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504; + + add_header 'Cache-Status' $upstream_cache_status always; + + proxy_pass http://127.0.0.1:{{node.port}}; + proxy_http_version 1.1; + + log_by_lua_block { + if ngx.var.upstream_cache_status == "MISS" then + -- we are using arg_host instead of host because we want to test a number of different hosts + cacher:add(ngx.var.arg_host, ngx.var.cache_key) + cacher:check_free_space(ngx) + end + } + } + } +} \ No newline at end of file diff --git a/proxy/tests/lru_purge.js b/proxy/tests/lru_purge.js new file mode 100644 index 00000000000..993ef37944f --- /dev/null +++ b/proxy/tests/lru_purge.js @@ -0,0 +1,81 @@ +const fs = require("fs-extra"); +const fetch = require("node-fetch"); +const setup = require("./util/setup"); +const { host } = require("../.."); + +describe("cacher", function () { + setup("./lru_purge.conf"); + + it( + "will automatically purge the cache when it fills up", + async function () { + const number_of_requests = 10000; + const hosts = {}; + + for (let i = 0; i < number_of_requests; i++) { + const host = randomhost(); + const res = await fetch( + this.origin + "/timestamp/" + i + "?host=" + host + ); + hosts[host] = i; + const text = await res.text(); + expect(res.status).toBe(200); + expect(parseInt(text)).toBeGreaterThan(0); + expect(res.headers.get("Cache-Status")).toBe("MISS"); + } + + // check that none of the files on disk are untracked by openresty shared dict + const { message, cache } = await this.inspectCache({ verbose: true }); + + // check that the missing hosts are the least recently used + const hosts_on_disk = Object.keys(cache); + + // there are no files on disk untracked in the lua shared dictionary + expect(message).toEqual("Cache is consistent"); + + // of the files on disk, the ones missing are the least recently used + const hosts_purged_from_disk = Object.keys(hosts) + .filter(host => !hosts_on_disk.includes(host)) + .sort((a, b) => { + return hosts[a] - hosts[b]; + }); + + const average_index_of_purged_hosts = + hosts_purged_from_disk.reduce((acc, host) => { + return acc + hosts[host]; + }, 0) / hosts_purged_from_disk.length; + + const average_index_of_hosts_on_disk = + hosts_on_disk.reduce((acc, host) => { + return acc + hosts[host]; + }, 0) / hosts_on_disk.length; + + console.log( + "average_index_of_purged_hosts", + average_index_of_purged_hosts + ); + console.log( + "average_index_of_hosts_on_disk", + average_index_of_hosts_on_disk + ); + expect(average_index_of_purged_hosts).toBeGreaterThan( + average_index_of_hosts_on_disk + ); + }, + 2 * 60 * 1000 // 2 minutes + ); +}); + +const letters = "abcdef"; +const host_length = 5; + +// will generate one of 7776 possible hosts +// e.g. aaaaa.com, aaaab.com, aaaac.com, etc. +function randomhost () { + let host = ""; + for (let i = 0; i < host_length; i++) { + host += letters[Math.floor(Math.random() * letters.length)]; + } + host += ".com"; + return host; +} diff --git a/proxy/tests/rehydrate.conf b/proxy/tests/rehydrate.conf new file mode 100644 index 00000000000..04b8420a028 --- /dev/null +++ b/proxy/tests/rehydrate.conf @@ -0,0 +1,81 @@ +user {{user}} {{group}}; + +events { + worker_connections 1000; + multi_accept on; +} + +http { + + log_format access_log_format '[$time_local] ID=$request_id STATUS=$status REQ_TIME=$request_time REQ_LEN=$request_length RES_LEN=$bytes_sent $scheme://$host$request_uri CACHE_STATUS=$sent_http_cache_status'; + + error_log {{{data_directory}}}/error.log debug; + access_log {{{data_directory}}}/access.log access_log_format; + + proxy_cache_path {{{cache_directory}}} levels=1:2 keys_zone=PROXY_CACHE:10m inactive=1y max_size=10g use_temp_path=off; + + lua_shared_dict cacher_dictionary 300k; + lua_package_path "{{{lua_directory}}}/?.lua;;"; + + init_by_lua_block { + cacher = (require "cacher").new() + cacher:set("cache_directory", "{{{cache_directory}}}") + cacher:set("shared_dictionary", ngx.shared.cacher_dictionary) + } + + init_worker_by_lua_block { + cacher:monitor_free_space(ngx, interval=1) + } + + server { + listen {{openresty.port}} default_server; + + location = /purge { + content_by_lua_block { + cacher:purge(ngx) + } + } + + location = /rehydrate { + content_by_lua_block { + local message = cacher:rehydrate(ngx) + ngx.say(message) + } + } + + location = /inspect { + content_by_lua_block { + cacher:inspect(ngx) + } + } + + + location = /health { + return 200 'Hello, World!'; + } + + location / { + open_file_cache off; + + set $cache_key $scheme://$host:$server_port$request_uri; + + proxy_cache PROXY_CACHE; + proxy_cache_valid 200 301 302 304 1y; + proxy_cache_key $cache_key; + proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504; + + add_header 'Cache-Status' $upstream_cache_status always; + + proxy_pass http://127.0.0.1:{{node.port}}; + proxy_http_version 1.1; + + # This registers a cached response in our dictionary of cached responses + # pass in the proxy_cache_key setting + log_by_lua_block { + if ngx.var.upstream_cache_status == "MISS" then + cacher:add(ngx.var.host, ngx.var.cache_key) + end + } + } + } +} \ No newline at end of file diff --git a/proxy/tests/rehydrate.js b/proxy/tests/rehydrate.js new file mode 100644 index 00000000000..ed255c77727 --- /dev/null +++ b/proxy/tests/rehydrate.js @@ -0,0 +1,95 @@ +const fs = require("fs-extra"); +const fetch = require("node-fetch"); +const setup = require("./util/setup"); + +describe("cacher", function () { + setup("./rehydrate.conf"); + + it("tells us when there's an invalid cache file", async function () { + const number_of_requests = 100; + + for (let i = 0; i < number_of_requests; i++) { + const res = await fetch(this.origin + "/timestamp/" + i); + const text = await res.text(); + expect(res.status).toBe(200); + expect(parseInt(text)).toBeGreaterThan(0); + expect(res.headers.get("Cache-Status")).toBe("MISS"); + } + // now we insert a cache files into the cache directory + // that are invalid + await fs.outputFile( + this.cache_directory + "/i/nv/no-key", + "invalid", + "utf8" + ); + + // now we insert a cache files into the cache directory + // that are invalid + await fs.outputFile( + this.cache_directory + "/i/nv/empty-key", + `KEY: + ...`, + "utf8" + ); + + // add a file containing 100 random bytes + await fs.outputFile( + this.cache_directory + "/i/nv/random-bytes", + new Array(100) + .fill(0) + .map(() => Math.random()) + .join(""), + "utf8" + ); + + // add an empty file + await fs.outputFile(this.cache_directory + "/i/nv/empty", "", "utf8"); + + const rehydrateResponse = await fetch(this.origin + "/rehydrate"); + const rehydrateText = await rehydrateResponse.text(); + + expect(rehydrateText.trim().split("\n")).toEqual([ + "i/nv/empty", + "i/nv/empty-key", + "i/nv/no-key", + "i/nv/random-bytes" + ]); + }); + + it( + "tells us when rehydration is successful", + async function () { + const number_of_requests = 100; + + for (let i = 0; i < number_of_requests; i++) { + const res = await fetch(this.origin + "/timestamp/" + i); + const text = await res.text(); + expect(res.status).toBe(200); + expect(parseInt(text)).toBeGreaterThan(0); + expect(res.headers.get("Cache-Status")).toBe("MISS"); + } + + const rehydrateResponse = await fetch(this.origin + "/rehydrate"); + const rehydrateText = await rehydrateResponse.text(); + + expect(rehydrateText.trim()).toEqual("OK"); + + const files = await this.listCache(); + + expect(files.length).toBe(number_of_requests); + + const inspectResponse = await fetch( + this.origin + "/inspect?host=127.0.0.1" + ); + const inspectText = await inspectResponse.text(); + + expect(inspectText.trim().split("\n").sort()).toEqual( + files.map(path => path.split("/").pop()).sort() + ); + + // checks for mismatches + expect(await this.inspectCache()).toEqual("Cache is consistent"); + }, + 1000 * 60 * 5 + ); +}); diff --git a/proxy/tests/util/inspect-cache.js b/proxy/tests/util/inspect-cache.js new file mode 100755 index 00000000000..7c82c1827df --- /dev/null +++ b/proxy/tests/util/inspect-cache.js @@ -0,0 +1,87 @@ +const fetch = require("node-fetch"); +const fs = require("fs-extra"); + +const main = async ( + origin = "http://127.0.0.1/inspect", + cache_directory = "/var/instance-ssd/cache", + only_host = null, + verbose = false +) => { + const top_level_directories = await fs.readdir(cache_directory); + const cache = {}; + let message = ""; + + // for each top level directory, read its contents + for (const directory of top_level_directories) { + const sub_directories = await fs.readdir(`${cache_directory}/${directory}`); + for (const sub_directory of sub_directories) { + const files = await fs.readdir( + `${cache_directory}/${directory}/${sub_directory}` + ); + for (const file of files) { + // open the file as a stream and read from it until we've either read 1mb + // or we have encountered the string 'KEY: '... newline + const stream = fs.createReadStream( + `${cache_directory}/${directory}/${sub_directory}/${file}`, + { encoding: "utf8" } + ); + let data = ""; + for await (const chunk of stream) { + data += chunk; + if (data.length > 1024 * 1024) break; + if (data.includes("KEY: ")) break; + } + const host_line = data.split("\n").find(line => line.includes("KEY: ")); + const parsed_url = require("url").parse(host_line.split("KEY: ")[1]); + // we don't want the port + const host = parsed_url.host.split(":")[0]; + if (only_host && host !== only_host) continue; + cache[host] = cache[host] || []; + cache[host].push(file); + } + } + } + + // go through each host in the directory and then request + // origin?host=host and compare the response to the cache + for (const host of Object.keys(cache).sort()) { + const response = await fetch(`${origin}?host=${host}`); + const data = await response.text(); + const cache_files = cache[host]; + const origin_files = data.trim().split("\n"); + const cache_files_set = new Set(cache_files); + const origin_files_set = new Set(origin_files); + const cache_only = cache_files.filter(file => !origin_files_set.has(file)); + const origin_only = origin_files.filter(file => !cache_files_set.has(file)); + + if (cache_only.length || origin_only.length) { + // these are files which exist on disk but are not tracked by the lua shared dict + if (cache_only.length) { + message += `${host} ${cache_only.join(", ")}` + "\n"; + } + // if (origin_only.length) { + // console.log(`Origin only: ${origin_only.join(", ")}`); + // } + } + } + + if (message.length === 0) { + message = "Cache is consistent"; + } + + if (verbose) { + return { + message, + cache + }; + } + + return message; +}; + +if (require.main === module) { + main(undefined, undefined, process.argv[2]) + .then(console.log) + .catch(console.error); +} +module.exports = main; diff --git a/proxy/tests/util/lookup-cache-file.js b/proxy/tests/util/lookup-cache-file.js new file mode 100644 index 00000000000..d71e22b1982 --- /dev/null +++ b/proxy/tests/util/lookup-cache-file.js @@ -0,0 +1,44 @@ +const fs = require("fs-extra"); +const fetch = require("node-fetch"); +const main = async cache_key_hash => { + const cache_directory = "/var/instance-ssd"; + // the cache file path is in the following format: $x/$y/$cache_key_hash + // where x is the last character of the cache_key_hash + // and y are the two characters before x + const cache_file_path = `${cache_directory}/${ + cache_key_hash[cache_key_hash.length - 1] + }/${cache_key_hash.slice(-3, -1)}/${cache_key_hash}`; + + const stream = fs.createReadStream(cache_file_path, { encoding: "utf8" }); + let data = ""; + for await (const chunk of stream) { + data += chunk; + if (data.length > 1024 * 1024) break; + // if data includes a blank line, break + if (data.includes("\r\n")) break; + } + const host_line = data.split("\n").find(line => line.includes("KEY: ")); + + console.log(host_line); + + const uri = host_line.split(" ")[1]; + const lines = data.split("\n"); + // headers are the lines after the 'KEY' line up until a blank line + const headers = lines.slice( + lines.indexOf(host_line) + 1, + lines.indexOf("\r") + ); + console.log(""); + console.log("FILE STAT: ", fs.statSync(cache_file_path)); + console.log("HEADERS", headers); + // we fetch the uri and check the response headers for 'Blot-Cache' which will be HIT or MISS + const response = await fetch(uri); + const cache_status = response.headers.get("Blot-Cache"); + console.log("cache_status", cache_status); +}; + +module.exports = main; + +if (require.main === module) { + main(process.argv[2]); +} diff --git a/proxy/tests/util/purge-from-disk.js b/proxy/tests/util/purge-from-disk.js new file mode 100644 index 00000000000..4f4e16dc054 --- /dev/null +++ b/proxy/tests/util/purge-from-disk.js @@ -0,0 +1,89 @@ +const fetch = require("node-fetch"); +const fs = require("fs-extra"); + +const main = async ( + origin = "http://127.0.0.1/purge", + cache_directory = "/var/instance-ssd", + host_match = null +) => { + if (!host_match) { + return console.log("no host match"); + } + const top_level_directories = await fs.readdir(cache_directory); + const hosts_to_purge = []; + + // for each top level directory, read its contents + for (const directory of top_level_directories) { + const sub_directories = await fs.readdir(`${cache_directory}/${directory}`); + for (const sub_directory of sub_directories) { + const files = await fs.readdir( + `${cache_directory}/${directory}/${sub_directory}` + ); + for (const file of files) { + // open the file as a stream and read from it until we've either read 1mb + // or we have encountered the string 'KEY: '... newline + const stream = fs.createReadStream( + `${cache_directory}/${directory}/${sub_directory}/${file}`, + { encoding: "utf8" } + ); + let data = ""; + for await (const chunk of stream) { + data += chunk; + if (data.length > 1024 * 1024) break; + if (data.includes("KEY: ")) break; + } + const host_line = data.split("\n").find(line => line.includes("KEY: ")); + const parsed_url = require("url").parse(host_line.split("KEY: ")[1]); + // we don't want the port + const host = parsed_url.host.split(":")[0]; + + // hostmatch is a string representing a host + // if it contains a *, we want to match it as a wildcard + // host_match = "*.blogsmith.com" + // host_match = "www.blogsmith.com" + // if host_match matches host, then push the host to hosts_to_purge + if (host_match.includes("*")) { + const regex = new RegExp( + host_match.replace(/\./g, "\\.").replace(/\*/g, ".*") + ); + if (regex.test(host)) hosts_to_purge.push(host); + } else { + if (host_match === host) hosts_to_purge.push(host); + } + } + } + } + + if (!hosts_to_purge.length) { + return console.log("no hosts to purge"); + } + + console.log("hosts to purge", hosts_to_purge); + + // break the list of hosts into chunks of 5 and then purge them + const chunks = hosts_to_purge.reduce( + (chunks, host) => { + if (chunks[chunks.length - 1].length === 5) chunks.push([]); + chunks[chunks.length - 1].push(host); + return chunks; + }, + [[]] + ); + + for (const chunk of chunks) { + const url = `${origin}?${chunk.map(host => `host=${host}`).join("&")}`; + console.log("fetching", url); + const response = await fetch(url); + const data = await response.text(); + console.log("----- response ----"); + console.log(data); + console.log(); + } +}; + +if (require.main === module) { + main(undefined, undefined, process.argv[2]) + .then(console.log) + .catch(console.error); +} +module.exports = main; diff --git a/proxy/tests/util/setup.js b/proxy/tests/util/setup.js new file mode 100644 index 00000000000..eb387904d53 --- /dev/null +++ b/proxy/tests/util/setup.js @@ -0,0 +1,154 @@ +const server = require("./upstream-server"); +const fetch = require("node-fetch"); +const child_process = require("child_process"); +const mustache = require("mustache"); +const { basename, resolve } = require("path"); +const fs = require("fs-extra"); +const { cache_directory } = require("../../.."); + +const CACHER_DIRECTORY = resolve(__dirname + "/../../"); +const DATA_DIRECTORY = CACHER_DIRECTORY + "/tests/data"; + +const config = { + openresty: { + port: 8787 + }, + node: { + port: 8788 + } +}; + +const inspectCache = require("./inspect-cache"); + +const startOpenresty = async (pathToConf, origin) => { + try { + const output = child_process.execSync( + __dirname + "/start-openresty.sh " + pathToConf + ); + // console.log(output.toString()); + } catch (e) { + console.log("stdout: ", e.stdout.toString()); + console.log("stderr: ", e.stderr.toString()); + throw new Error("Failed to start openresty"); + } + + let connected = false; + + while (!connected) { + try { + // this will not be cached + // console.log("Checking " + origin + "/health"); + // this seems to hang sometimes, set a timeout of 1 second + await fetch(origin + "/health", { timeout: 1000 }); + connected = true; + } catch (e) { + // console.log("Openresty not started yet"); + } + } +}; + +const stopOpenresty = async () => { + try { + const output = child_process.execSync(__dirname + "/stop-openresty.sh"); + // console.log(output.toString()); + } catch (e) {} +}; + +module.exports = configFile => { + beforeEach(async function () { + await fs.emptyDir(DATA_DIRECTORY); + + const origin = `http://127.0.0.1:${config.openresty.port}`; + + this.origin = origin; + + const cache_directory = DATA_DIRECTORY + "/cache"; + const configInput = CACHER_DIRECTORY + "/tests/" + configFile; + const configPath = DATA_DIRECTORY + "/" + basename(configFile); + + this.cache_directory = cache_directory; + + const result = mustache.render(await fs.readFile(configInput, "utf8"), { + ...config, + user: process.env.BLOT_OPENRESTY_TEST_USER || "David", + group: process.env.BLOT_OPENRESTY_TEST_GROUP || "staff", + lua_directory: CACHER_DIRECTORY + "/conf", + data_directory: DATA_DIRECTORY, + cache_directory + }); + + await fs.outputFile(configPath, result, "utf8"); + + await fs.mkdir(DATA_DIRECTORY + "/cache"); + + await stopOpenresty(); + + await startOpenresty(configPath, this.origin); + + this.server = await server({ port: config.node.port }); + + this.inspectCache = ({ verbose = false, host = null } = {}) => + inspectCache(origin + "/inspect", cache_directory, host, verbose); + + this.restartOpenresty = async () => { + // get the pid of the current openresty process + const masterpid = () => + child_process + .execSync( + "ps aux | grep openresty | grep 'master process' | awk '{print $2}'" + ) + .toString() + .trim(); + + const pidBefore = masterpid(); + + await stopOpenresty(); + await startOpenresty(configPath, origin); + + // check that the pid has changed + const pidAfter = masterpid(); + + if (!pidBefore) { + throw new Error("Openresty was not running"); + } + + if (!pidAfter) { + throw new Error("Openresty did not start"); + } + + if (pidBefore === pidAfter) { + throw new Error("Openresty did not restart"); + } + }; + + this.listCache = async function listCache ({ watch = true } = {}) { + const files = []; + + const list = async function (dir) { + const stat = await fs.stat(dir); + if (stat.isDirectory()) { + const children = await fs.readdir(dir); + for (const child of children) { + await list(dir + "/" + child); + } + } else { + files.push(dir); + } + }; + + await list(cache_directory); + + // wait for the cache to be populated + while (files.length === 0 && watch) { + await list(cache_directory); + } + + return files; + }; + }); + + afterEach(async function () { + await stopOpenresty(); + await this.server.close(); + }); +}; diff --git a/proxy/tests/util/start-openresty.sh b/proxy/tests/util/start-openresty.sh new file mode 100755 index 00000000000..ae86bb329e7 --- /dev/null +++ b/proxy/tests/util/start-openresty.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +CONFIG=$1 + +if [ -z "$CONFIG" ]; then + echo "No config file specified" + exit 1 +fi + +# determine the path to the openresty executable +# on my mac it will be present in the PATH already, but on github actions +# we need to use: /usr/local/openresty/bin/openresty +OPENRESTY=$(which openresty) + +if [ -z "$OPENRESTY" ]; then + OPENRESTY="/usr/local/openresty/bin/openresty" +fi + +echo "Starting openresty as root with config $CONFIG" + +sudo $OPENRESTY -c $CONFIG \ No newline at end of file diff --git a/proxy/tests/util/stop-openresty.sh b/proxy/tests/util/stop-openresty.sh new file mode 100755 index 00000000000..7311d88405e --- /dev/null +++ b/proxy/tests/util/stop-openresty.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# kill all openresty processes +PIDS=$(ps -ef | grep nginx | grep -v grep | awk '{print $2}') + +if [ -z "$PIDS" ]; then + echo "No openresty processes running" +else + +# for each PID, kill it +for PID in $PIDS + do + echo "Killing nginx process $PID" + sudo kill -9 $PID + done +fi \ No newline at end of file diff --git a/proxy/tests/util/upstream-server.js b/proxy/tests/util/upstream-server.js new file mode 100644 index 00000000000..5ebe62eee79 --- /dev/null +++ b/proxy/tests/util/upstream-server.js @@ -0,0 +1,44 @@ +const express = require("express"); +const app = express(); +const sharp = require("sharp"); + +app.get("/", (req, res) => { + res.send("Hello Node!"); +}); + +// dynamic route +app.use("/timestamp", (req, res) => { + res.send(`${Date.now()}`); +}); + +app.use("/gzip", (req, res) => { + res.send("abc ".repeat(1024)); +}); + +app.get("/:filename.png", (req, res) => { + const { filename } = req.params; + + res.set("Content-Type", "image/png"); + + sharp({ + create: { + width: 400, + height: 200, + channels: 4, + background: { r: 255, g: 0, b: 0, alpha: 0.5 } // transparent red + } + }) + .png() + .toBuffer() + .then(data => { + res.send(data); + }); +}); + +module.exports = ({ port }) => { + return new Promise((resolve, reject) => { + const server = app.listen(port, () => { + resolve(server); + }); + }); +};