From 765ddcef8763c8c7e465b3bcee6ab59d385f2134 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Fri, 17 Apr 2026 14:09:37 -0700 Subject: [PATCH 1/5] init 2.0.0 --- manticore/versions/2.0.0/Chart.yaml | 13 + manticore/versions/2.0.0/README.md | 258 ++++++++ .../versions/2.0.0/templates/_helpers.tpl | 266 ++++++++ .../versions/2.0.0/templates/domain.yaml | 34 ++ .../versions/2.0.0/templates/identity.yaml | 66 ++ .../versions/2.0.0/templates/policy.yaml | 99 +++ .../versions/2.0.0/templates/secret.yaml | 321 ++++++++++ .../2.0.0/templates/volumeset-shared.yaml | 11 + .../versions/2.0.0/templates/volumeset.yaml | 15 + .../versions/2.0.0/templates/workload.yaml | 577 ++++++++++++++++++ manticore/versions/2.0.0/values.yaml | 219 +++++++ 11 files changed, 1879 insertions(+) create mode 100644 manticore/versions/2.0.0/Chart.yaml create mode 100644 manticore/versions/2.0.0/README.md create mode 100644 manticore/versions/2.0.0/templates/_helpers.tpl create mode 100644 manticore/versions/2.0.0/templates/domain.yaml create mode 100644 manticore/versions/2.0.0/templates/identity.yaml create mode 100644 manticore/versions/2.0.0/templates/policy.yaml create mode 100644 manticore/versions/2.0.0/templates/secret.yaml create mode 100644 manticore/versions/2.0.0/templates/volumeset-shared.yaml create mode 100644 manticore/versions/2.0.0/templates/volumeset.yaml create mode 100644 manticore/versions/2.0.0/templates/workload.yaml create mode 100644 manticore/versions/2.0.0/values.yaml diff --git a/manticore/versions/2.0.0/Chart.yaml b/manticore/versions/2.0.0/Chart.yaml new file mode 100644 index 00000000..a7526e95 --- /dev/null +++ b/manticore/versions/2.0.0/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v2 +name: manticore +description: Distributed Manticore Search cluster with intelligent orchestration. + +type: application +version: 2.0.0 +appVersion: "25.0.0" + +annotations: + created: "2026-01-05" + lastModified: "2026-04-17" + category: "search" + createsGvc: false \ No newline at end of file diff --git a/manticore/versions/2.0.0/README.md b/manticore/versions/2.0.0/README.md new file mode 100644 index 00000000..fe07cae8 --- /dev/null +++ b/manticore/versions/2.0.0/README.md @@ -0,0 +1,258 @@ +# Manticore Search Cluster + +Deploys a distributed Manticore Search cluster on Control Plane with automatic Galera-based replication, zero-downtime data imports, multi-table support, backup/restore, and a web UI for cluster management. + +## Architecture + +The template deploys several components that work together: + +- **Manticore Workload** - Stateful replicas running Manticore searchd, each with a sidecar agent for local operations +- **Orchestrator API** - REST API that coordinates cluster-wide operations (initialization, imports, repairs, backups) +- **Orchestrator Job** - Cron workload for on-demand job execution +- **UI** - Web dashboard for monitoring and managing the cluster + +The orchestrator handles cluster initialization, coordinates imports across all replicas using a dual-slot (A/B) system for zero-downtime swaps, and provides automatic repair for split-brain scenarios. All replicas stay in sync via Galera cluster replication. + +## Prerequisites + +1. **S3 Bucket** - Create an S3 bucket to store your CSV source files +2. **Control Plane Cloud Account** - Follow the [Create a Cloud Account](https://docs.controlplane.com/guides/create-cloud-account) guide to establish trust between Control Plane and your AWS account + +## Installation + +1. **Configure S3 access** in `values.yaml`: + ```yaml + buckets: + cloudAccountName: your-cloud-account + awsPolicyRefs: + - aws::AmazonS3ReadOnlyAccess # or your custom policy + sourceBucket: your-bucket-name + ``` + +2. **Define your tables**: + ```yaml + tables: + - name: products + csvPath: imports/products/data.csv + config: + haStrategy: noerrors # HA strategy for distributed queries; 'noerrors' skips agents that return errors + agentRetryCount: 3 # Number of times to retry failed agent connections + clusterMain: false # Set to true to replicate the main table across all cluster nodes + memLimit: 2G # Memory limit for indexer during import (max = 2G) + hasHeader: true # Set to true if the CSV file includes a header row + schema: + columns: + - name: title + type: field + - name: price + type: attr_float + ``` + +3. **Generate an authentication token**: + ```bash + openssl rand -base64 32 + ``` + Set this in `orchestrator.agent.token`. This bearer token secures all internal API communication between components. + +**Note:** After installation, the cluster will be initialized but tables will be empty until you run an import. See [Operations](#operations) below. + +## Authentication + +All internal communication is secured with the bearer token set in `orchestrator.agent.token`. This token is shared across the orchestrator, agents, and UI. + +- Must be set before deployment +- Should be cryptographically random (use `openssl rand -base64 32`) +- Rotating requires redeploying all components + +**Security note:** The UI injects this token automatically, so anyone with network access to the UI can perform admin operations. Restrict access by setting `orchestrator.ui.allowExternalAccess: false` or using a domain with authentication. + +## Configuration Reference + +### Core Settings + +| Path | Description | Default | +|------|-------------|---------| +| `buckets.cloudAccountName` | AWS Cloud Account name | - | +| `buckets.sourceBucket` | S3 bucket with CSV files | - | +| `manticore.clusterName` | Galera cluster name | `manticore` | +| `manticore.autoscaling.minScale` | Minimum replicas | `3` | +| `manticore.autoscaling.maxScale` | Maximum replicas | `4` | + +### Table Configuration + +Each entry in `tables[]` supports: + +| Field | Description | +|-------|-------------| +| `name` | Table name | +| `csvPath` | Path to CSV in S3 bucket, or a list of paths for multi-segment tables (see [Multi-Segment Tables](#multi-segment-tables)) | +| `config.haStrategy` | HA strategy: `noerrors`, `nodeads`, etc. | +| `config.agentRetryCount` | Retry count for distributed queries | +| `config.clusterMain` | Replicate main tables across cluster | +| `config.segmentCount` | Number of distributed table segments; must match the number of entries in `csvPath` (default: `1`) | +| `config.importMethod` | Import method: `indexer` or `sql` | +| `config.charsetTable` | Manticore `charset_table` tokenization preset (e.g., `non_cont`) — omit to use the Manticore default | +| `config.memLimit` | Memory limit for indexer operations (e.g., `2G`) | +| `config.hasHeader` | Whether the CSV file has a header row (`true`/`false`) | +| `schema.columns` | Column definitions (see column types below) | + +### Column Types + +| Type | Description | +|------|-------------| +| `field` | Full-text searchable field | +| `field_string` | Full-text field (string variant) | +| `attr_uint` | Unsigned integer attribute | +| `attr_bigint` | Big integer attribute | +| `attr_float` | Float attribute | +| `attr_bool` | Boolean attribute | +| `attr_string` | String attribute (not full-text indexed) | +| `attr_timestamp` | Timestamp attribute | +| `attr_multi` | Multi-value integer attribute | +| `attr_multi_64` | Multi-value 64-bit integer attribute | +| `attr_json` | JSON attribute | + +**Note**: If column 1 is numeric, it's used as the document ID (don't declare it). If not numeric, an ID is auto-generated. + +### Orchestrator Settings + +| Path | Description | Default | +|------|-------------|---------| +| `orchestrator.schedule` | Cron schedule for imports | `0 * * * *` | +| `orchestrator.action` | Action: `init`, `import`, `health`, `repair` | `import` | +| `orchestrator.tableName` | Table to import | - | +| `orchestrator.suspend` | Start suspended | `true` | +| `orchestrator.agent.token` | Bearer token for auth | **required** | + +## Multi-Segment Tables + +Large datasets can be split across multiple CSV files and imported as a distributed table with multiple independent segments. Manticore fans queries across all segments automatically. + +Set `csvPath` to a list of S3 paths and set `segmentCount` to match the number of entries: + +```yaml +tables: + - name: addresses + csvPath: + - large-file/part1.csv + - large-file/part2.csv + config: + segmentCount: 2 # must match the number of csvPath entries + importMethod: indexer + memLimit: 2G + hasHeader: true + schema: + columns: + - name: street_name + type: field +``` + +`segmentCount` must equal the number of items in `csvPath`. The template will fail at render time with a descriptive error if they don't match. + +When a table has multiple segments, a backup backs up **all segments** as one file. Restores on multi-segment tables will restore all segments on the table. + +## Operations + +Operations can be triggered via the **Orchestrator UI** or the **Control Plane CLI/API**. + +### Via Orchestrator UI + +The web dashboard provides controls for: +- **Import, Backup and Restore** - Select a table and trigger a coordinated import, backup, or restore process +- **Repair** - Recover the cluster from split-brain scenarios +- **Monitoring** - View cluster health, replica status, and table details + +### Via Control Plane + +Run the orchestrator cron workload to execute operations: + +```bash +# Trigger an import +cpln workload run-cron {release-name}-orchestrator-job --gvc {gvc-name} + +# Trigger a repair (set ACTION=repair on the workload first) +cpln workload run-cron {release-name}-orchestrator-job --gvc {gvc-name} +``` + +## Load Testing + +Enable k6 load testing to validate search performance: + +```yaml +loadTest: + enabled: true + vus: 10 + duration: "5m" + query: + index: products + query: + match: + "*": "test" +``` + +Trigger via Control Plane: +```bash +cpln workload run-cron {release-name}-load-test-controller --gvc {gvc-name} +``` + +Or set `loadTest.controller.schedule` to run on a cron schedule. + +## Backup & Restore + +Backup and restore is available for both **delta** (real-time updates) and **main** (full indexed dataset) tables. Backups are stored as compressed archives in S3. + +### Prerequisites + +1. **S3 Bucket** for storing backups (can be shared with or separate from source data) +2. **IAM Policy** with `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`, `s3:ListBucket` permissions on the bucket +3. **Cloud Account** with the above policy attached + +### Configuration + +Enable backups in `values.yaml`: + +```yaml +orchestrator: + backup: + enabled: true + cloudAccountName: my-backup-cloud-account + s3Bucket: my-backup-bucket + s3Policy: + - my-backup-policy + s3Region: us-east-1 + prefix: manticore-backups + schedules: [ # Automated backup schedules (optional) + {"table": "products", "type": "delta", "schedule": "0 2 * * *"}, + {"table": "products", "type": "main", "schedule": "0 3 * * 0"} + ] +``` + +### Usage + +**Via Orchestrator UI:** +- **Backup**: Select a type (delta/main) and click "Backup" +- **Restore**: Select a type, choose a backup file from the list, and confirm +- **Rotate Main**: After a main restore, swap the active slot + +**Via API:** +```bash +# Backup +curl -X POST "https://{orchestrator-api-url}/api/backup" \ + -H "Authorization: Bearer {token}" \ + -H "Content-Type: application/json" \ + -d '{"tableName": "products", "type": "delta"}' + +# List backups +curl "https://{orchestrator-api-url}/api/backups/files?tableName=products" \ + -H "Authorization: Bearer {token}" + +# Restore +curl -X POST "https://{orchestrator-api-url}/api/restore" \ + -H "Authorization: Bearer {token}" \ + -H "Content-Type: application/json" \ + -d '{"tableName": "products", "type": "delta", "filename": "products_delta-2024-01-28T22-50-49Z.tar.gz"}' +``` + +## Links +- [Manticore Search Docs](https://manual.manticoresearch.com/) +- [Orchestrator, Agent, UI and Backup source code](https://github.com/controlplane-com/manticore-orchestrator) \ No newline at end of file diff --git a/manticore/versions/2.0.0/templates/_helpers.tpl b/manticore/versions/2.0.0/templates/_helpers.tpl new file mode 100644 index 00000000..d378f13d --- /dev/null +++ b/manticore/versions/2.0.0/templates/_helpers.tpl @@ -0,0 +1,266 @@ +{{/* Resource Naming */}} + +{{/* +Manticore Workload Name +*/}} +{{- define "manticore.name" -}} +{{- printf "%s-manticore" .Release.Name }} +{{- end }} + +{{/* +Manticore Orchestrator Job Workload Name +*/}} +{{- define "manticore.orchestratorJobName" -}} +{{- printf "%s-orchestrator-job" .Release.Name }} +{{- end }} + +{{/* +Manticore Orchestrator API Workload Name +*/}} +{{- define "manticore.orchestratorAPIName" -}} +{{- printf "%s-orchestrator-api" .Release.Name }} +{{- end }} + +{{/* +Manticore UI Workload Name +*/}} +{{- define "manticore.UIName" -}} +{{- printf "%s-ui" .Release.Name }} +{{- end }} + +{{/* +Manticore Backup Workload Name +*/}} +{{- define "manticore.backupName" -}} +{{- printf "%s-manticore-backup" .Release.Name }} +{{- end }} + +{{/* +Manticore Load Test Workload Name +*/}} +{{- define "manticore.loadTestName" -}} +{{- printf "%s-load-test" .Release.Name }} +{{- end }} + +{{/* +Manticore Load Test Controller Workload Name +*/}} +{{- define "manticore.loadTestControllerName" -}} +{{- printf "%s-load-test-controller" .Release.Name }} +{{- end }} + +{{/* +Manticore Secret Config Name +*/}} +{{- define "manticore.secretConfigName" -}} +{{- printf "%s-manticore-config" .Release.Name }} +{{- end }} + +{{/* +Manticore Secret Startup Name +*/}} +{{- define "manticore.secretStartupName" -}} +{{- printf "%s-manticore-startup" .Release.Name }} +{{- end }} + +{{/* +Manticore Secret Schema Config Name +*/}} +{{- define "manticore.secretSchemaConfigName" -}} +{{- printf "%s-manticore-schema" .Release.Name }} +{{- end }} + +{{/* +Manticore Secret Agent Token Name +*/}} +{{- define "manticore.secretAgentTokenName" -}} +{{- printf "%s-manticore-agent-token" .Release.Name }} +{{- end }} + +{{/* +Manticore Secret K6 Script Name +*/}} +{{- define "manticore.secretK6ScriptName" -}} +{{- printf "%s-manticore-k6-script" .Release.Name }} +{{- end }} + +{{/* +Manticore Identity Name +*/}} +{{- define "manticore.identityName" -}} +{{- printf "%s-manticore-identity" .Release.Name }} +{{- end }} + +{{/* +Manticore Orchestrator Identity Name +*/}} +{{- define "manticore.orchestratorIdentityName" -}} +{{- printf "%s-manticore-orchestrator-identity" .Release.Name }} +{{- end }} + +{{/* +Manticore Orchestrator Job Identity Name +*/}} +{{- define "manticore.orchestratorJobIdentityName" -}} +{{- printf "%s-manticore-orchestrator-job-identity" .Release.Name }} +{{- end }} + +{{/* +Manticore Load Test Identity Name +*/}} +{{- define "manticore.loadTestIdentityName" -}} +{{- printf "%s-manticore-load-test-identity" .Release.Name }} +{{- end }} + +{{/* +Manticore Load Test Controller Identity Name +*/}} +{{- define "manticore.loadTestControllerIdentityName" -}} +{{- printf "%s-manticore-load-test-controller-identity" .Release.Name }} +{{- end }} + +{{/* +Manticore Backup Identity Name +*/}} +{{- define "manticore.backupIdentityName" -}} +{{- printf "%s-manticore-backup-identity" .Release.Name }} +{{- end }} + +{{/* +Manticore Config Policy Name +*/}} +{{- define "manticore.configPolicyName" -}} +{{- printf "%s-manticore-config-policy" .Release.Name }} +{{- end }} + +{{/* +Manticore Exec Policy Name +*/}} +{{- define "manticore.execPolicyName" -}} +{{- printf "%s-manticore-exec-policy" .Release.Name }} +{{- end }} + +{{/* +Manticore Orchestrator Policy Name +*/}} +{{- define "manticore.orchestratorPolicyName" -}} +{{- printf "%s-manticore-orchestrator-policy" .Release.Name }} +{{- end }} + +{{/* +Manticore Load Test Policy Name +*/}} +{{- define "manticore.loadTestPolicyName" -}} +{{- printf "%s-manticore-load-test-policy" .Release.Name }} +{{- end }} + +{{/* +Manticore Load Test Controller Policy Name +*/}} +{{- define "manticore.loadTestControllerPolicyName" -}} +{{- printf "%s-manticore-load-test-controller-policy" .Release.Name }} +{{- end }} + +{{/* +Manticore Volume Set Name +*/}} +{{- define "manticore.volumeName" -}} +{{- printf "%s-manticore-vs" .Release.Name }} +{{- end }} + +{{/* +Manticore Shared Volume Set Name +*/}} +{{- define "manticore.sharedVolumeName" -}} +{{- printf "%s-manticore-vs-shared" .Release.Name }} +{{- end }} + + +{{/* Functions */}} + +{{/* +Generate JSON mapping of table names to CSV paths for orchestrator. +csvPath accepts a single string or a list for multi-segment tables. +Output (single): {"addresses":"imports/addresses/data.csv"} +Output (multi): {"addresses":["imports/addresses/data_1.csv","imports/addresses/data_2.csv"]} +*/}} +{{- define "manticore.tablesConfigJSON" -}} +{{- $config := dict -}} +{{- range . -}} +{{- $_ := set $config .name .csvPath -}} +{{- end -}} +{{- $config | toJson -}} +{{- end }} + +{{/* +Validate that each table's csvPath length matches its config.segmentCount. +csvPath may be a single string (segmentCount must be 1) or a list (length must equal segmentCount). +*/}} +{{- define "manticore.validateTables" -}} +{{- range .Values.tables -}} +{{- $tableName := .name -}} +{{- $segmentCount := .config.segmentCount | int -}} +{{- if kindIs "slice" .csvPath -}} + {{- $csvCount := len .csvPath -}} + {{- if ne $csvCount $segmentCount -}} + {{- fail (printf "Table %q: csvPath has %d entries but segmentCount is %d — they must match." $tableName $csvCount $segmentCount) -}} + {{- end -}} +{{- else -}} + {{- if ne $segmentCount 1 -}} + {{- fail (printf "Table %q: csvPath is a single string but segmentCount is %d — it must be 1 when csvPath is a single value." $tableName $segmentCount) -}} + {{- end -}} +{{- end -}} +{{- end -}} +{{- end }} + +{{/* +Calculate total load test duration in seconds (duration + buffer) +Parses duration strings like "5m", "1h", "30s" +*/}} +{{- define "loadTest.totalDurationSeconds" -}} +{{- $duration := .Values.loadTest.duration -}} +{{- $buffer := .Values.loadTest.controller.testDurationBuffer | int -}} +{{- $seconds := 0 -}} +{{- if hasSuffix "s" $duration -}} + {{- $seconds = trimSuffix "s" $duration | int -}} +{{- else if hasSuffix "m" $duration -}} + {{- $seconds = mul (trimSuffix "m" $duration | int) 60 -}} +{{- else if hasSuffix "h" $duration -}} + {{- $seconds = mul (trimSuffix "h" $duration | int) 3600 -}} +{{- end -}} +{{- add $seconds $buffer -}} +{{- end }} + + +{{/* Labeling */}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "manticore.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "manticore.tags" -}} +helm.sh/chart: {{ include "manticore.chart" . }} +{{ include "manticore.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.cpln.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.cpln.io/managed-by: {{ .Release.Service }} +cpln/marketplace: "true" +cpln/marketplace-template: manticore +cpln/marketplace-template-version: {{ .Chart.Version }} +cpln/marketplace-gvc: {{ .Values.global.cpln.gvc }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "manticore.selectorLabels" -}} +app.cpln.io/name: {{ .Release.Name }} +app.cpln.io/instance: {{ .Release.Name }} +{{- end }} \ No newline at end of file diff --git a/manticore/versions/2.0.0/templates/domain.yaml b/manticore/versions/2.0.0/templates/domain.yaml new file mode 100644 index 00000000..aad6906a --- /dev/null +++ b/manticore/versions/2.0.0/templates/domain.yaml @@ -0,0 +1,34 @@ +# ============================================================================= +# External Domain (Optional) +# ============================================================================= +# Routes /api/* to orchestrator-api, /* to UI +{{- if .Values.domain.enabled }} +kind: domain +name: {{ .Values.domain.name }} +description: External domain for Manticore cluster UI and API +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + dnsMode: {{ .Values.domain.dnsMode }} + gvcLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }} + acceptAllHosts: false + ports: + - number: 443 + protocol: http2 + cors: + allowOrigins: + - exact: '*' + allowMethods: + - GET + - POST + - OPTIONS + allowHeaders: + - '*' + allowCredentials: true + routes: + - prefix: /api/ + workloadLink: //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.orchestratorAPIName" . }} + port: 8080 + - prefix: / + workloadLink: //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.UIName" . }} + port: 3000 +{{- end }} diff --git a/manticore/versions/2.0.0/templates/identity.yaml b/manticore/versions/2.0.0/templates/identity.yaml new file mode 100644 index 00000000..cad0607e --- /dev/null +++ b/manticore/versions/2.0.0/templates/identity.yaml @@ -0,0 +1,66 @@ +# ============================================================================= +# Workload Identities +# ============================================================================= +# Manticore identity - S3 access via AWS Cloud Account +kind: identity +name: {{ include "manticore.identityName" . }} +description: Manticore workload identity for secret access +tags: {{- include "manticore.tags" . | nindent 4 }} + +--- +# Orchestrator identity - secret access + CPLN API for cluster operations + s3 access +kind: identity +name: {{ include "manticore.orchestratorIdentityName" . }} +description: Orchestrator identity for secrets, CPLN API, and S3 access +tags: {{- include "manticore.tags" . | nindent 4 }} +{{- if .Values.orchestrator.backup.enabled }} +aws: + cloudAccountLink: //cloudaccount/{{ .Values.orchestrator.backup.cloudAccountName }} + policyRefs: + {{- range .Values.orchestrator.backup.s3Policy }} + - {{ . }} + {{- end }} +{{- end }} + +--- +# Orchestrator job identity - secret access + S3 access for cron operations +kind: identity +name: {{ include "manticore.orchestratorJobIdentityName" . }} +description: Orchestrator job identity for secrets and S3 access +tags: {{- include "manticore.tags" . | nindent 4 }} +aws: + cloudAccountLink: //cloudaccount/{{ .Values.buckets.cloudAccountName }} + policyRefs: + {{- range .Values.buckets.awsPolicyRefs }} + - {{ . }} + {{- end }} + +{{- if .Values.orchestrator.backup.enabled }} +--- +# Backup identity - S3 access for backups +kind: identity +name: {{ include "manticore.backupIdentityName" . }} +description: Manticore backup identity for S3 access +tags: {{- include "manticore.tags" . | nindent 4 }} +aws: + cloudAccountLink: //cloudaccount/{{ .Values.orchestrator.backup.cloudAccountName }} + policyRefs: + {{- range .Values.orchestrator.backup.s3Policy }} + - {{ . }} + {{- end }} +{{- end }} + +{{- if .Values.loadTest.enabled }} +--- +# Load test identity - access to k6 script secret +kind: identity +name: {{ include "manticore.loadTestIdentityName" . }} +description: Load test workload identity for script access +tags: {{- include "manticore.tags" . | nindent 4 }} +--- +# Load test controller identity - workload scaling permissions +kind: identity +name: {{ include "manticore.loadTestControllerIdentityName" . }} +description: Controller identity for scaling load-test workload +tags: {{- include "manticore.tags" . | nindent 4 }} +{{- end }} diff --git a/manticore/versions/2.0.0/templates/policy.yaml b/manticore/versions/2.0.0/templates/policy.yaml new file mode 100644 index 00000000..78c046f7 --- /dev/null +++ b/manticore/versions/2.0.0/templates/policy.yaml @@ -0,0 +1,99 @@ +# ============================================================================= +# Access Policies +# ============================================================================= +# Secret access policy - grants reveal on config secrets +kind: policy +name: {{ include "manticore.configPolicyName" . }} +description: Secret access for manticore and orchestrator workloads +tags: {{- include "manticore.tags" . | nindent 4 }} +bindings: + - permissions: + - reveal + principalLinks: + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.identityName" . }} + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorIdentityName" . }} + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorJobIdentityName" . }} + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.backupIdentityName" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "manticore.secretConfigName" . }} + - //secret/{{ include "manticore.secretStartupName" . }} + - //secret/{{ include "manticore.secretSchemaConfigName" . }} + - //secret/{{ include "manticore.secretAgentTokenName" . }} +targetQuery: + kind: secret + fetch: items + spec: + match: all + terms: [] +--- +# Cron execution policy - allows triggering orchestrator jobs +kind: policy +name: {{ include "manticore.execPolicyName" . }} +description: Permission to trigger orchestrator cron workload executions +tags: {{- include "manticore.tags" . | nindent 4 }} +bindings: + - permissions: + - view + - edit + - exec.runCronWorkload + principalLinks: + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.identityName" . }} + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorIdentityName" . }} +targetKind: workload +targetLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.orchestratorJobName" . }} + - //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.name" . }} + {{- if .Values.orchestrator.backup.enabled }} + - //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.backupName" . }} + {{- end }} + +--- +# Workload view policy - allows orchestrator to read Manticore workload config +kind: policy +name: {{ include "manticore.orchestratorPolicyName" . }} +description: Permission to view Manticore workload configuration +tags: {{- include "manticore.tags" . | nindent 4 }} +bindings: + - permissions: + - view + - edit + principalLinks: + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorJobIdentityName" . }} + - /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.backupIdentityName" . }} +targetKind: workload +targetLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.name" . }} + +{{- if .Values.loadTest.enabled }} +--- +# Load test script access +kind: policy +name: {{ include "manticore.loadTestPolicyName" . }} +description: K6 workload access to load test script secret +tags: {{- include "manticore.tags" . | nindent 4 }} +bindings: + - permissions: + - use + - reveal + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.loadTestIdentityName" . }} +targetKind: secret +targetLinks: + - //secret/{{ include "manticore.secretK6ScriptName" . }} +--- +# Load test controller policy - allows scaling k6 workload +kind: policy +name: {{ include "manticore.loadTestControllerPolicyName" . }} +description: Controller permission to scale load-test workload +tags: {{- include "manticore.tags" . | nindent 4 }} +bindings: + - permissions: + - manage + - edit + principalLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.loadTestControllerIdentityName" . }} +targetKind: workload +targetLinks: + - //gvc/{{ .Values.global.cpln.gvc }}/workload/{{ include "manticore.loadTestName" . }} +{{- end }} diff --git a/manticore/versions/2.0.0/templates/secret.yaml b/manticore/versions/2.0.0/templates/secret.yaml new file mode 100644 index 00000000..95c45322 --- /dev/null +++ b/manticore/versions/2.0.0/templates/secret.yaml @@ -0,0 +1,321 @@ +{{- include "manticore.validateTables" . -}} +# ============================================================================= +# Manticore Search Configuration +# ============================================================================= +# Base searchd config (startup script generates runtime config with IP-bound listeners) +kind: secret +name: {{ include "manticore.secretConfigName" . }} +description: Manticore searchd base configuration +tags: {{- include "manticore.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + searchd { + listen = 9306:mysql + listen = 9308:http + listen = 9312 + data_dir = /var/lib/manticore + binlog_path = /var/lib/manticore/binlog + log = /dev/stdout + query_log = /dev/stdout + pid_file = /var/run/searchd.pid + seamless_rotate = 1 + preopen_tables = 1 + } +--- +# ============================================================================= +# Schema Registry +# ============================================================================= +# Table schemas for agent (creates RT delta tables, parses CSV, builds distributed tables) +kind: secret +name: {{ include "manticore.secretSchemaConfigName" . }} +description: Schema registry for multi-table configuration +tags: {{- include "manticore.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + # Schema Registry (YAML format, read by agent) + # See README.md for column types and configuration options + {{- range .Values.tables }} + + {{ .name }}: + config: + {{- .config | toYaml | nindent 8 }} + schema: + columns: + {{- range .schema.columns }} + - name: {{ .name }} + type: {{ .type }} + {{- end }} + {{- end }} +--- +# ============================================================================= +# Manticore Startup Script +# ============================================================================= +# Generates runtime config, starts searchd, handles graceful shutdown +# Cluster init (bootstrap/join) handled by agent via orchestrator API +kind: secret +name: {{ include "manticore.secretStartupName" . }} +description: Manticore searchd startup and shutdown handler +tags: {{- include "manticore.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + #!/usr/bin/env bash + set -euo pipefail + + # ============================================================================= + # CONFIGURATION + # ============================================================================= + + CLUSTER_NAME="{{ .Values.manticore.clusterName }}" + MYSQL_PORT=9306 + HTTP_PORT=9308 + REPL_PORT=9312 + WORKLOAD_NAME="$(echo "${HOSTNAME}" | sed 's/-[0-9]*$//')" + REPLICA_INDEX="$(echo "${HOSTNAME}" | awk -F'-' '{print $NF}')" + LOCATION=$(basename "${CPLN_LOCATION}") + # Internal DNS format: {workloadName}-{replicaIndex}.{workloadName} + NODE0_FQDN="${WORKLOAD_NAME}-0.${WORKLOAD_NAME}" + NODE0_ADDR="${NODE0_FQDN}:${REPL_PORT}" + + echo "============================================" + echo "Manticore Startup" + echo "============================================" + echo "Hostname: ${HOSTNAME}" + echo "Replica Index: ${REPLICA_INDEX}" + echo "Cluster: ${CLUSTER_NAME}" + echo "Node 0 FQDN: ${NODE0_FQDN}" + echo "============================================" + echo "" + + # Create required directories + echo "Creating directories..." + mkdir -p /var/lib/manticore/binlog + mkdir -p /var/lib/manticore/data + echo "Directories created." + + RUNTIME_CONFIG="/var/lib/manticore/manticore-runtime.conf" + + # ============================================================================= + # HELPER FUNCTIONS + # ============================================================================= + + mysql_exec() { + local host="${1:-127.0.0.1}" + shift || true + mysql --protocol=tcp -h "${host}" -P "${MYSQL_PORT}" -N -B "$@" + } + + wait_for_manticore() { + echo "Waiting for Manticore MySQL port ${MYSQL_PORT}..." + for i in $(seq 1 60); do + if mysql --protocol=tcp -h 127.0.0.1 -P "${MYSQL_PORT}" -e "SELECT 1" >/dev/null 2>&1; then + echo "Manticore is ready." + return 0 + fi + echo " [$i/60] not ready yet..." + sleep 1 + done + echo "ERROR: Manticore did not become ready on ${MYSQL_PORT}" + return 1 + } + + # ============================================================================= + # GRACEFUL SHUTDOWN HANDLER + # ============================================================================= + + shutdown() { + echo "" + echo "============================================" + echo "SIGTERM received: graceful shutdown" + echo "============================================" + + echo "Stopping searchd gracefully..." + if searchd --stopwait --config "${RUNTIME_CONFIG}"; then + echo "searchd stopped gracefully" + else + echo "searchd --stopwait failed, falling back to kill" + if [[ -n "${SEARCHD_PID:-}" ]] && kill -0 "$SEARCHD_PID" >/dev/null 2>&1; then + kill "$SEARCHD_PID" || true + for _ in $(seq 1 30); do + kill -0 "$SEARCHD_PID" >/dev/null 2>&1 || break + sleep 1 + done + fi + fi + + # Keep manticore.json on all replicas so they remember cluster state and + # table associations. This allows IST (incremental sync) instead of full SST + # when rejoining, and ensures local tables are recognized as cluster tables. + if [[ "${REPLICA_INDEX}" != "0" ]]; then + echo "Requesting cluster node refresh on replica-0..." + mysql_exec "${NODE0_FQDN}" -e "ALTER CLUSTER ${CLUSTER_NAME} UPDATE nodes" || true + fi + + echo "Shutdown sequence complete." + echo "============================================" + } + + trap shutdown TERM INT + + # ============================================================================= + # START MANTICORE + # ============================================================================= + + echo "" + echo "============================================" + echo "Preparing searchd configuration..." + echo "============================================" + + # Get the pod's IP address and FQDN for replication binding + MY_IP=$(hostname -i | awk '{print $1}') + MY_FQDN="${WORKLOAD_NAME}-${REPLICA_INDEX}.${WORKLOAD_NAME}" + echo "Pod IP address: ${MY_IP}" + echo "Pod FQDN: ${MY_FQDN}" + + # Create runtime config with dynamic replication listener + echo "Generating runtime config: ${RUNTIME_CONFIG}" + + echo "searchd {" > "${RUNTIME_CONFIG}" + echo " listen = 127.0.0.1:9306:mysql" >> "${RUNTIME_CONFIG}" + echo " listen = ${MY_IP}:9306:mysql" >> "${RUNTIME_CONFIG}" + echo " listen = 127.0.0.1:9308:http" >> "${RUNTIME_CONFIG}" + echo " listen = ${MY_IP}:9308:http" >> "${RUNTIME_CONFIG}" + echo " listen = 127.0.0.1:9312" >> "${RUNTIME_CONFIG}" + echo " listen = ${MY_IP}:9312" >> "${RUNTIME_CONFIG}" + echo " listen = ${MY_IP}:9313-9320:replication" >> "${RUNTIME_CONFIG}" + echo " node_address = ${MY_FQDN}" >> "${RUNTIME_CONFIG}" + echo " data_dir = /var/lib/manticore" >> "${RUNTIME_CONFIG}" + echo " binlog_path = /var/lib/manticore/binlog" >> "${RUNTIME_CONFIG}" + echo " log = /dev/stdout" >> "${RUNTIME_CONFIG}" + echo " query_log = /dev/stdout" >> "${RUNTIME_CONFIG}" + echo " pid_file = /var/run/searchd.pid" >> "${RUNTIME_CONFIG}" + echo " seamless_rotate = 1" >> "${RUNTIME_CONFIG}" + echo " preopen_tables = 1" >> "${RUNTIME_CONFIG}" + echo " server_id = ${REPLICA_INDEX}" >> "${RUNTIME_CONFIG}" + echo "}" >> "${RUNTIME_CONFIG}" + + echo "Runtime config generated:" + cat "${RUNTIME_CONFIG}" + echo "" + + echo "============================================" + echo "Starting searchd..." + echo "============================================" + + echo "" + echo "Launching searchd in background..." + searchd --config "${RUNTIME_CONFIG}" --nodetach 2>&1 & + SEARCHD_PID="$!" + echo "searchd launched with PID: ${SEARCHD_PID}" + + sleep 2 + + if ! kill -0 "${SEARCHD_PID}" 2>/dev/null; then + echo "ERROR: searchd exited immediately! Check config." + exit 1 + fi + + echo "searchd is running. Waiting for MySQL port..." + wait_for_manticore + + # ============================================================================= + # CLUSTER STATUS CHECK (cluster setup is handled by orchestrator init) + # ============================================================================= + + echo "" + echo "============================================" + echo "Cluster Status" + echo "============================================" + + # Check if already part of a cluster (from preserved state in manticore.json) + # Wait a few seconds for cluster module to load state + sleep 3 + CLUSTER_STATUS=$(mysql_exec 127.0.0.1 -e "SHOW STATUS LIKE 'cluster_${CLUSTER_NAME}_status'" 2>/dev/null || echo "") + + if echo "${CLUSTER_STATUS}" | grep -qE "(primary|synced)"; then + echo "Already part of cluster ${CLUSTER_NAME}." + else + echo "Not part of any cluster. Run orchestrator init to bootstrap/join cluster." + fi + + # ============================================================================= + # KEEP CONTAINER ALIVE + # ============================================================================= + + echo "" + echo "============================================" + echo "Startup complete" + echo "============================================" + echo "searchd PID: ${SEARCHD_PID}" + echo "============================================" + echo "" + echo "Import operations are handled by the agent sidecar." + echo "Use the orchestrator workload for coordinated imports." + echo "" + + wait "${SEARCHD_PID}" +--- +# ============================================================================= +# Agent Authentication Token +# ============================================================================= +# Bearer token for orchestrator/agent/UI communication +# Generate with: openssl rand -base64 32 +kind: secret +name: {{ include "manticore.secretAgentTokenName" . }} +description: Bearer token for orchestrator-agent authentication +tags: {{- include "manticore.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: {{ required "agent.token is required" .Values.orchestrator.agent.token }} + +# ============================================================================= +# K6 Load Test Script +# ============================================================================= +{{- if .Values.loadTest.enabled }} +--- +# Generated k6 script from loadTest.* values +kind: secret +name: {{ include "manticore.secretK6ScriptName" . }} +description: K6 load test script for Manticore search +tags: {{- include "manticore.tags" . | nindent 4 }} +type: opaque +data: + encoding: plain + payload: |- + import http from 'k6/http'; + import { check } from 'k6'; + import { Rate } from 'k6/metrics'; + + const errorRate = new Rate('errors'); + + export const options = { + vus: {{ .Values.loadTest.vus }}, + duration: '{{ .Values.loadTest.duration }}', + thresholds: { + 'http_req_duration': ['p(95)<{{ .Values.loadTest.thresholds.p95ResponseTime }}'], + 'http_req_failed': ['rate<{{ .Values.loadTest.thresholds.errorRate }}'], + }, + }; + + const BASE_URL = 'http://{{ include "manticore.name" . }}.{{ .Values.global.cpln.gvc }}.cpln.local:{{ .Values.loadTest.target.port }}'; + + const QUERY = JSON.stringify({{ .Values.loadTest.query | toJson }}); + + export default function () { + const res = http.post(`${BASE_URL}/{{ .Values.loadTest.target.endpoint }}`, QUERY, { + headers: { 'Content-Type': 'application/json' }, + }); + + const success = check(res, { + 'status is 200': (r) => r.status === 200, + }); + + errorRate.add(!success); + } +{{- end }} diff --git a/manticore/versions/2.0.0/templates/volumeset-shared.yaml b/manticore/versions/2.0.0/templates/volumeset-shared.yaml new file mode 100644 index 00000000..77934a40 --- /dev/null +++ b/manticore/versions/2.0.0/templates/volumeset-shared.yaml @@ -0,0 +1,11 @@ +# ============================================================================= +# Shared Volumeset +# ============================================================================= +# Shared storage across replicas and orchestrator +kind: volumeset +name: {{ include "manticore.sharedVolumeName" . }} +description: Shared storage across Manticore replicas and orchestrator +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + fileSystemType: shared + initialCapacity: {{ .Values.manticore.sharedVolumeset.capacity }} diff --git a/manticore/versions/2.0.0/templates/volumeset.yaml b/manticore/versions/2.0.0/templates/volumeset.yaml new file mode 100644 index 00000000..7935d42b --- /dev/null +++ b/manticore/versions/2.0.0/templates/volumeset.yaml @@ -0,0 +1,15 @@ +# ============================================================================= +# Manticore Volumeset +# ============================================================================= +# Persistent storage per replica (data, binlog, cluster state) +kind: volumeset +name: {{ include "manticore.volumeName" . }} +description: Persistent storage for Manticore data and cluster state +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + fileSystemType: ext4 + initialCapacity: {{ .Values.manticore.volumeset.capacity }} + performanceClass: general-purpose-ssd + snapshots: + createFinalSnapshot: true + retentionDuration: 7d \ No newline at end of file diff --git a/manticore/versions/2.0.0/templates/workload.yaml b/manticore/versions/2.0.0/templates/workload.yaml new file mode 100644 index 00000000..8acc6b6d --- /dev/null +++ b/manticore/versions/2.0.0/templates/workload.yaml @@ -0,0 +1,577 @@ +# ============================================================================= +# Manticore Search Workload (Stateful) +# ============================================================================= +# Each replica runs manticore (searchd) + agent sidecar for orchestrator coordination +kind: workload +name: {{ include "manticore.name" . }} +description: Manticore search cluster with agent sidecar +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + type: stateful + containers: + - name: manticore + cpu: {{ .Values.manticore.resources.cpu | quote }} + image: {{ .Values.manticore.image }} + command: "/bin/bash" + args: + - "/usr/local/bin/start.sh" + inheritEnv: false + memory: {{ .Values.manticore.resources.memory | quote }} + metrics: + path: "/metrics" + port: 9308 + ports: + - number: 9306 + protocol: tcp + - number: 9308 + protocol: http + - number: 9312 + protocol: tcp + # Galera replication ports + - number: 9313 + protocol: tcp + - number: 9314 + protocol: tcp + - number: 9315 + protocol: tcp + - number: 9316 + protocol: tcp + - number: 9317 + protocol: tcp + - number: 9318 + protocol: tcp + - number: 9319 + protocol: tcp + - number: 9320 + protocol: tcp + readinessProbe: + failureThreshold: 6 + initialDelaySeconds: 30 + periodSeconds: 15 + successThreshold: 1 + tcpSocket: + port: 9306 + timeoutSeconds: 10 + volumes: + - path: /var/lib/manticore + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "manticore.volumeName" . }} + - path: /mnt/shared + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "manticore.sharedVolumeName" . }} + - path: /etc/manticore/manticore.conf + recoveryPolicy: retain + uri: cpln://secret/{{ include "manticore.secretConfigName" . }}.payload + - path: /etc/manticore/schema.conf + recoveryPolicy: retain + uri: cpln://secret/{{ include "manticore.secretSchemaConfigName" . }}.payload + - path: /usr/local/bin/start.sh + recoveryPolicy: retain + uri: cpln://secret/{{ include "manticore.secretStartupName" . }}.payload + - name: agent + cpu: {{ .Values.orchestrator.agent.resources.cpu | quote }} + minCpu: {{ .Values.orchestrator.agent.resources.minCpu | default .Values.orchestrator.agent.resources.cpu | quote }} + image: {{ .Values.orchestrator.agent.image }}:{{ .Values.orchestrator.agent.version }} + inheritEnv: false + memory: {{ .Values.orchestrator.agent.resources.memory | quote }} + minMemory: {{ .Values.orchestrator.agent.resources.minMemory | default .Values.orchestrator.agent.resources.memory | quote }} + ports: + - number: 8080 + protocol: http + env: + - name: MYSQL_HOST + value: "127.0.0.1" + - name: MYSQL_PORT + value: "9306" + - name: SCHEMA_FILE + value: "/etc/manticore/schema.conf" + - name: CLUSTER_NAME + value: "{{ .Values.manticore.clusterName }}" + - name: LISTEN_ADDR + value: ":8080" + - name: AUTH_TOKEN + value: "cpln://secret/{{ include "manticore.secretAgentTokenName" . }}.payload" + # Cluster recovery settings + - name: WORKLOAD_NAME + value: "{{ include "manticore.name" . }}" + - name: DATA_DIR + value: "/var/lib/manticore" + - name: MAX_SCALE + value: "{{ .Values.manticore.autoscaling.maxScale }}" + - name: RECOVERY_MAX_RETRIES + value: "{{ .Values.orchestrator.agent.recovery.maxRetries | default 5 }}" + - name: RECOVERY_INITIAL_BACKOFF_SEC + value: "{{ .Values.orchestrator.agent.recovery.initialBackoffSec | default 5 }}" + - name: RECOVERY_MAX_BACKOFF_SEC + value: "{{ .Values.orchestrator.agent.recovery.maxBackoffSec | default 60 }}" + - name: REPLICATION_PORT + value: "9312" + - name: IMPORT_BATCH_SIZE + value: "{{ .Values.orchestrator.agent.import.batchSize | default 1000 }}" + - name: ORCHESTRATOR_API_URL + value: "http://{{ include "manticore.orchestratorAPIName" . }}.{{ .Values.global.cpln.gvc }}.cpln.local:8080" + readinessProbe: + failureThreshold: 20 + initialDelaySeconds: 10 + periodSeconds: 15 + successThreshold: 1 + httpGet: + path: /api/ready + port: 8080 + timeoutSeconds: 5 + volumes: + - path: /etc/manticore/schema.conf + recoveryPolicy: retain + uri: cpln://secret/{{ include "manticore.secretSchemaConfigName" . }}.payload + - path: /mnt/shared + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "manticore.sharedVolumeName" . }} + - path: /var/lib/manticore + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "manticore.volumeName" . }} + defaultOptions: + autoscaling: + maxScale: {{ .Values.manticore.autoscaling.maxScale }} + metric: {{ .Values.manticore.autoscaling.metric }} + minScale: {{ .Values.manticore.autoscaling.minScale }} + scaleToZeroDelay: {{ .Values.manticore.autoscaling.scaleToZeroDelay }} + target: {{ .Values.manticore.autoscaling.target }} + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 5 + firewallConfig: + external: + inboundAllowCIDR: [] + outboundAllowHostname: [] + internal: + inboundAllowType: {{ .Values.manticore.firewall.internalAccess.type }} + {{- if .Values.manticore.firewall.internalAccess.workloads }} + inboundAllowWorkload: {{ .Values.manticore.firewall.internalAccess.workloads | toYaml | nindent 8 }} + {{- end }} + identityLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.identityName" . }} + loadBalancer: + direct: + enabled: false + ports: [] + replicaDirect: true + rolloutOptions: + maxSurgeReplicas: {{ .Values.manticore.rolloutOptions.maxSurgeReplicas }} + maxUnavailableReplicas: '{{ .Values.manticore.rolloutOptions.maxUnavailableReplicas }}' + minReadySeconds: {{ .Values.manticore.rolloutOptions.minReadySeconds }} + scalingPolicy: {{ .Values.manticore.rolloutOptions.scalingPolicy }} + terminationGracePeriodSeconds: {{ .Values.manticore.rolloutOptions.terminationGracePeriodSeconds }} + supportDynamicTags: false +--- +# ============================================================================= +# Orchestrator Cron Workload +# ============================================================================= +# Executes scheduled/manual actions: init, import, health, repair +# Starts suspended by default (trigger via UI/API) +kind: workload +name: {{ include "manticore.orchestratorJobName" . }} +description: Manticore orchestrator for scheduled/manual cluster operations +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + type: cron + containers: + - name: orchestrator + cpu: {{ .Values.orchestrator.resources.cpu | quote }} + image: {{ .Values.orchestrator.image }}:{{ .Values.orchestrator.version }} + inheritEnv: false + memory: {{ .Values.orchestrator.resources.memory | quote }} + env: + - name: MODE + value: "cli" + - name: ACTION + value: "{{ .Values.orchestrator.action }}" + - name: REPLICA_COUNT + value: "{{ .Values.manticore.autoscaling.minScale }}" + - name: AGENT_PORT + value: "8080" + - name: WORKLOAD_NAME + value: "{{ include "manticore.name" . }}" + - name: GVC + value: "{{ .Values.global.cpln.gvc }}" + - name: LOCATION + value: "{{ .Values.global.cpln.location }}" + - name: TABLE_NAME + value: "{{ .Values.orchestrator.tableName }}" + - name: TABLES_CONFIG + value: '{{ include "manticore.tablesConfigJSON" .Values.tables }}' + - name: STATE_FILE + value: "/tmp/orchestrator_state.json" + - name: AUTH_TOKEN + value: "cpln://secret/{{ include "manticore.secretAgentTokenName" . }}.payload" + - name: LOG_LEVEL + value: "{{ .Values.orchestrator.logLevel }}" + - name: IMPORT_MEM_LIMIT + value: "{{ .Values.orchestrator.importMemLimit }}" + - name: INDEXER_WORK_DIR + value: "/mnt/s3/indexer-temp" + - name: S3_MOUNT + value: "/mnt/s3" + - name: SHARED_VOLUME_MOUNT + value: "/mnt/shared" + volumes: + - path: /mnt/shared + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "manticore.sharedVolumeName" . }} + - path: /mnt/s3 + uri: s3://{{ .Values.buckets.sourceBucket }} + recoveryPolicy: retain + defaultOptions: + autoscaling: + maxScale: 1 + minScale: 1 + capacityAI: false + debug: false + suspend: {{ .Values.orchestrator.suspend }} + timeoutSeconds: {{ .Values.orchestrator.timeoutSeconds }} + firewallConfig: + external: + inboundAllowCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + outboundAllowHostname: [] + internal: + inboundAllowType: none + identityLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorJobIdentityName" . }} + job: + schedule: {{ .Values.orchestrator.schedule | quote }} + concurrencyPolicy: Forbid + restartPolicy: Never + activeDeadlineSeconds: {{ .Values.orchestrator.activeDeadlineSeconds }} + supportDynamicTags: false +--- +# ============================================================================= +# Orchestrator API Workload (Standard) +# ============================================================================= +# REST API for cluster coordination (init, import, repair, cluster status) +# Backend for UI dashboard, communicates with agents via bearer token +kind: workload +name: {{ include "manticore.orchestratorAPIName" . }} +description: Manticore orchestrator REST API for cluster coordination +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + type: standard + containers: + - name: api + cpu: {{ .Values.orchestrator.api.resources.cpu | quote }} + image: {{ .Values.orchestrator.api.image }}:{{ .Values.orchestrator.api.version }} + inheritEnv: false + memory: {{ .Values.orchestrator.api.resources.memory | quote }} + port: 8080 + readinessProbe: + httpGet: + path: /api/status + port: 8080 + periodSeconds: 10 + failureThreshold: 3 + env: + - name: MODE + value: "server" + - name: LISTEN_ADDR + value: ":8080" + - name: REPLICA_COUNT + value: "{{ .Values.manticore.autoscaling.minScale }}" + - name: AGENT_PORT + value: "8080" + - name: WORKLOAD_NAME + value: "{{ include "manticore.name" . }}" + # CPLN_GVC, CPLN_LOCATION auto-injected by Control Plane + - name: TABLES_CONFIG + value: '{{ include "manticore.tablesConfigJSON" .Values.tables }}' + - name: STATE_FILE + value: "/tmp/orchestrator_state.json" + - name: AUTH_TOKEN + value: "cpln://secret/{{ include "manticore.secretAgentTokenName" . }}.payload" + - name: LOG_LEVEL + value: "{{ .Values.orchestrator.logLevel }}" + # CPLN_TOKEN, CPLN_ORG, CPLN_GVC auto-injected by Control Plane + - name: ORCHESTRATOR_WORKLOAD + value: "{{ include "manticore.orchestratorJobName" . }}" + - name: IMPORT_POLL_INTERVAL + value: "{{ .Values.orchestrator.api.importPollInterval | default "30s" }}" + - name: IMPORT_POLL_TIMEOUT + value: "{{ .Values.orchestrator.api.importPollTimeout | default "2h" }}" + {{- if .Values.orchestrator.backup.enabled }} + - name: BACKUP_SCHEDULES + value: '{{ .Values.orchestrator.backup.schedules | toJson }}' + - name: BACKUP_BUCKET + value: {{ .Values.orchestrator.backup.s3Bucket }} + - name: BACKUP_PREFIX + value: {{ .Values.orchestrator.backup.prefix }} + - name: BACKUP_PROVIDER + value: aws + - name: BACKUP_REGION + value: {{ .Values.orchestrator.backup.s3Region }} + {{- end }} + defaultOptions: + autoscaling: + maxScale: {{ .Values.orchestrator.api.autoscaling.maxScale }} + minScale: {{ .Values.orchestrator.api.autoscaling.minScale }} + metric: {{ .Values.orchestrator.api.autoscaling.metric }} + target: {{ .Values.orchestrator.api.autoscaling.target }} + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 30 + firewallConfig: + external: + inboundAllowCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + internal: + inboundAllowType: same-gvc + identityLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorIdentityName" . }} + supportDynamicTags: false +--- +# ============================================================================= +# Web UI Workload (Standard) +# ============================================================================= +# Dashboard for cluster monitoring, table status, imports, and repairs +kind: workload +name: {{ include "manticore.UIName" . }} +description: Manticore cluster management dashboard +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + type: standard + containers: + - name: ui + cpu: {{ .Values.orchestrator.ui.resources.cpu | quote }} + image: {{ .Values.orchestrator.ui.image }}:{{ .Values.orchestrator.ui.version }} + inheritEnv: false + memory: {{ .Values.orchestrator.ui.resources.memory | quote }} + port: 3000 + env: + - name: ORCHESTRATOR_API_URL + value: "http://{{ include "manticore.orchestratorAPIName" . }}.{{ .Values.global.cpln.gvc }}.cpln.local:8080" + - name: ORCHESTRATOR_AUTH_TOKEN + value: "cpln://secret/{{ include "manticore.secretAgentTokenName" . }}.payload" + - name: PORT + value: "3000" + readinessProbe: + httpGet: + path: / + port: 3000 + periodSeconds: 10 + failureThreshold: 3 + defaultOptions: + autoscaling: + maxScale: {{ .Values.orchestrator.ui.autoscaling.maxScale }} + minScale: {{ .Values.orchestrator.ui.autoscaling.minScale }} + metric: {{ .Values.orchestrator.ui.autoscaling.metric }} + target: {{ .Values.orchestrator.ui.autoscaling.target }} + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 30 + firewallConfig: + external: + inboundAllowCIDR: {{ if .Values.orchestrator.ui.allowExternalAccess }}[ "0.0.0.0/0" ]{{ else }}[ ]{{ end }} + outboundAllowCIDR: [] + internal: + inboundAllowType: same-gvc + identityLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.orchestratorIdentityName" . }} + supportDynamicTags: false + +{{- if .Values.orchestrator.backup.enabled }} +--- +# ============================================================================= +# Backup Workload (Cron) +# ============================================================================= +# Cron job for logical backups on delta table to S3 bucket +kind: workload +name: {{ include "manticore.backupName" . }} +description: manticore backup +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + type: cron + containers: + - name: backup + cpu: {{ .Values.orchestrator.backup.resources.cpu | quote }} + env: + - name: ACTION + value: backup + - name: TYPE + value: delta + - name: BACKUP_BUCKET + value: {{ .Values.orchestrator.backup.s3Bucket }} + - name: BACKUP_PREFIX + value: {{ .Values.orchestrator.backup.prefix }} + - name: BACKUP_PROVIDER + value: aws + - name: BACKUP_REGION + value: {{ .Values.buckets.awsRegion }} + - name: DATASET + value: {{ .Values.orchestrator.backup.dataSet }} + - name: MANTICORE_HOST + value: "{{ include "manticore.name" . }}" + - name: MANTICORE_PORT + value: "9306" + - name: AUTH_TOKEN + value: "cpln://secret/{{ include "manticore.secretAgentTokenName" . }}.payload" + image: {{ .Values.orchestrator.backup.image }}:{{ .Values.orchestrator.backup.version }} + inheritEnv: false + memory: {{ .Values.orchestrator.backup.resources.memory | quote }} + ports: + - number: 8080 + protocol: http + volumes: + - path: /mnt/shared + recoveryPolicy: retain + uri: cpln://volumeset/{{ include "manticore.sharedVolumeName" . }} + defaultOptions: + autoscaling: + maxScale: 1 + metric: disabled + minScale: 1 + capacityAI: false + debug: false + multiZone: + enabled: false + suspend: true + timeoutSeconds: 5 + firewallConfig: + external: + inboundAllowCIDR: [] + inboundBlockedCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + outboundAllowHostname: [] + outboundAllowPort: [] + outboundBlockedCIDR: [] + internal: + inboundAllowType: same-gvc + inboundAllowWorkload: [] + identityLink: //gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.backupIdentityName" . }} + job: + concurrencyPolicy: Forbid + historyLimit: 5 + restartPolicy: Never + schedule: '0 2 * * *' + activeDeadlineSeconds: {{ .Values.orchestrator.backup.activeDeadlineSeconds }} + localOptions: [] + supportDynamicTags: false +{{- end }} + +# ============================================================================= +# K6 Load Test Workload (Standard) +# ============================================================================= +# Runs k6 load tests (starts at scale 0, scaled up by controller) +{{- if .Values.loadTest.enabled }} +--- +kind: workload +name: {{ include "manticore.loadTestName" . }} +description: K6 load test runner for Manticore search +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + type: standard + containers: + - name: k6 + image: {{ .Values.loadTest.image }} + cpu: {{ .Values.loadTest.resources.cpu | quote }} + memory: {{ .Values.loadTest.resources.memory | quote }} + inheritEnv: false + command: /bin/sh + args: + - '-c' + - >- + k6 run + --vus {{ .Values.loadTest.vus }} + --duration {{ .Values.loadTest.duration }} + {{- if .Values.loadTest.rps }} + --rps {{ .Values.loadTest.rps }} + {{- end }} + /scripts/payload + volumes: + - path: /scripts + recoveryPolicy: retain + uri: 'cpln://secret/{{ include "manticore.secretK6ScriptName" . }}.payload' + defaultOptions: + autoscaling: + maxScale: 0 + minScale: 0 + scaleToZeroDelay: 30 + target: 95 + capacityAI: false + debug: false + suspend: false + timeoutSeconds: 3600 + firewallConfig: + external: + inboundAllowCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + internal: + inboundAllowType: none + identityLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.loadTestIdentityName" . }} + supportDynamicTags: false +--- +# ============================================================================= +# Load Test Controller (Cron) +# ============================================================================= +# Scales k6 workload up, waits for test duration, then scales back to 0 +# Suspended if no schedule (manual trigger only) +kind: workload +name: {{ include "manticore.loadTestControllerName" . }} +description: Controller for triggering and managing load tests +tags: {{- include "manticore.tags" . | nindent 4 }} +spec: + type: cron + containers: + - name: controller + image: {{ .Values.loadTest.controller.image }} + cpu: '0.25' + memory: 256Mi + inheritEnv: false + command: /bin/sh + args: + - '-c' + - | + set -e + WORKLOAD="{{ include "manticore.loadTestName" . }}" + REPLICAS="{{ .Values.loadTest.replicas }}" + + echo "Scaling $WORKLOAD to $REPLICAS replicas..." + curl -sf -X PATCH \ + -H "Authorization: Bearer $CPLN_TOKEN" \ + -H "Content-Type: application/json" \ + -d "{\"spec\":{\"defaultOptions\":{\"autoscaling\":{\"minScale\":$REPLICAS,\"maxScale\":$REPLICAS}}}}" \ + "http://api.cpln.io/org/$CPLN_ORG/gvc/$CPLN_GVC/workload/$WORKLOAD" + + echo "Waiting for test duration + buffer..." + sleep {{ include "loadTest.totalDurationSeconds" . }} + + echo "Scaling $WORKLOAD back to 0..." + curl -sf -X PATCH \ + -H "Authorization: Bearer $CPLN_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"spec":{"defaultOptions":{"autoscaling":{"minScale":0,"maxScale":0}}}}' \ + "http://api.cpln.io/org/$CPLN_ORG/gvc/$CPLN_GVC/workload/$WORKLOAD" + + echo "Load test complete." + defaultOptions: + autoscaling: + maxScale: 1 + minScale: 1 + capacityAI: false + debug: false + suspend: {{ if .Values.loadTest.controller.schedule }}false{{ else }}true{{ end }} + firewallConfig: + external: + inboundAllowCIDR: [] + outboundAllowCIDR: + - 0.0.0.0/0 + internal: + inboundAllowType: none + identityLink: /org/{{ .Values.global.cpln.org }}/gvc/{{ .Values.global.cpln.gvc }}/identity/{{ include "manticore.loadTestControllerIdentityName" . }} + job: + schedule: '{{ .Values.loadTest.controller.schedule | default "0 0 1 1 *" }}' + concurrencyPolicy: Forbid + restartPolicy: Never + historyLimit: 10 + activeDeadlineSeconds: 7200 + supportDynamicTags: false +{{- end }} diff --git a/manticore/versions/2.0.0/values.yaml b/manticore/versions/2.0.0/values.yaml new file mode 100644 index 00000000..12b7e1f2 --- /dev/null +++ b/manticore/versions/2.0.0/values.yaml @@ -0,0 +1,219 @@ +# ============================================================================= +# AWS Cloud Account and S3 Configuration +# ============================================================================= +# See README.md for Cloud Account and IAM policy setup instructions. +buckets: + cloudAccountName: my-cloud-account # Name of your configured Cloud Account + awsPolicyRefs: # IAM policies for S3 access + - aws::AmazonS3ReadOnlyAccess # Note: if using a custom policy, omit the aws:: prefix as this is only for AWS managed policies + awsRegion: us-east-1 # Region of your S3 bucket + sourceBucket: my-bucket # S3 bucket containing files to import + +# ============================================================================= +# Tables Configuration +# ============================================================================= +# See README.md for table structure, column types, and config options. +# Add/remove tables as needed. +tables: + - name: addresses + csvPath: + - manticore-imports/addresses/data.csv + config: + haStrategy: noerrors + agentRetryCount: 3 + clusterMain: false + segmentCount: 1 + charsetTable: non_cont + memLimit: 2G + hasHeader: true + schema: + columns: + - name: address_id + type: attr_uint + - name: street_number + type: attr_uint + - name: street_name + type: field + - name: city + type: field + - name: county + type: field + - name: state + type: field + - name: postal_code + type: field + - name: country + type: field + - name: latitude + type: attr_float + - name: longitude + type: attr_float + +# ============================================================================= +# Manticore Search Configuration +# ============================================================================= +manticore: + image: manticoresearch/manticore:25.0.0 + clusterName: manticore # Galera cluster name + resources: + cpu: 4 + memory: 8Gi + volumeset: + capacity: 200 # GB per replica + sharedVolumeset: + capacity: 100 # GB shared across replicas and orchestrator + + # minScale = replica count used by orchestrator for coordination + autoscaling: + minScale: 3 + maxScale: 4 + metric: rps # rps for stateful, cpu for standard + target: 100 + scaleToZeroDelay: 300 + + rolloutOptions: + maxSurgeReplicas: 25% + maxUnavailableReplicas: '0' + minReadySeconds: 10 + scalingPolicy: OrderedReady + terminationGracePeriodSeconds: 60 + + # IMPORTANT: same-gvc required for Galera replication + firewall: + internalAccess: + type: same-gvc + workloads: [] + +# ============================================================================= +# Orchestrator Configuration +# ============================================================================= +orchestrator: + version: v6.0.0 + image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-api + logLevel: debug # debug, info, warn, error + resources: + cpu: 1 + memory: 2Gi + + # Cron job settings + schedule: "0 * * * *" # Cron schedule (default = every hour) + action: import # init, import, health, repair + tableName: addresses # Must match a name in tables[] + suspend: true # Start suspended (trigger via UI/API) + timeoutSeconds: 900 # Container timeout (seconds, default 15 minutes) + importMemLimit: 2G # Memory limit for import jobs + activeDeadlineSeconds: 14400 # Max job runtime (seconds, default 4 hours) + + # Orchestrator API + api: + version: v6.0.0 + image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-api + logLevel: debug + importPollInterval: 30s + importPollTimeout: 2h + resources: + cpu: 0.25 + memory: 256Mi + autoscaling: + maxScale: 3 + minScale: 2 + metric: cpu + target: 80 + + # Agent sidecar + agent: + version: v6.0.0 + image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-agent + # REQUIRED: Generate with `openssl rand -base64 32` + token: "6Gl5uO9KkKAh1u+ymoBW98WCtjTFpljuhpLdKb+tNAA=" + resources: + cpu: 250m + minCpu: 100m + memory: 512Mi + minMemory: 128Mi + import: + batchSize: 20000 # Rows per INSERT statement + recovery: + maxRetries: 5 # Retry attempts for cluster recovery + initialBackoffSec: 5 # Initial delay between retries + maxBackoffSec: 60 # Max backoff delay (exponential) + + # Web UI - See README.md "Authentication" section for security notes + ui: + version: v6.0.0 + image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-ui + resources: + cpu: 0.25 + memory: 0.25Gi + allowExternalAccess: true # false = internal GVC access only + autoscaling: + maxScale: 2 + minScale: 1 + metric: cpu + target: 80 + + # Backup Configuration (Optional) - Cron job runs logical backup on delta table to S3 bucket. + backup: + enabled: false + version: v6.0.0 + image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-backup + cloudAccountName: my-backup-cloud-account + s3Bucket: my-backup-bucket # S3 bucket for backups + s3Policy: # IAM policies for S3 access + - my-backup-policy # Custom policy created in S3 setup instructions + s3Region: us-east-1 + dataSet: addresses # Data set to back up + prefix: manticore-backups # S3 prefix/folder for backups + schedules: [ + {"table":"addresses","type":"delta","schedule":"0 2 * * *"}, # Daily at 2am UTC + {"table":"addresses","type":"main","schedule":"0 2 1 * *"} # Monthly full backup on 1st at 2am UTC + ] + activeDeadlineSeconds: 14400 # Max job runtime (seconds, default 4 hours) + resources: + cpu: 1 + memory: 1Gi + +# ============================================================================= +# Domain Configuration (Optional) +# ============================================================================= +# Routes /api/* to orchestrator-api, everything else to UI. +domain: + enabled: false + name: "" # FQDN, e.g., manticore.example.com + dnsMode: cname # cname (subdomains) or ns (zone delegation) + +# ============================================================================= +# Load Testing Configuration (Optional) +# ============================================================================= +loadTest: + enabled: false + image: grafana/k6:0.47.0 + resources: + cpu: 0.5 + memory: 512Mi + + vus: 10 # Virtual users + duration: "5m" # Test duration (e.g., 30s, 5m, 1h) + rps: null # Target RPS (null = unlimited) + replicas: 1 # Number of k6 pods to spawn + + controller: + image: alpine/curl # Alpine image with curl pre-installed + schedule: "" # Cron expression (empty = manual only) + testDurationBuffer: 60 # Seconds added to duration before scale-down + + target: + port: 9308 # Manticore HTTP API port + endpoint: search # "search" or "sql" + + # Full JSON body for /search endpoint + query: + index: addresses + query: + match: + "*": "test" + limit: 10 + + thresholds: + p95ResponseTime: 500 # ms + errorRate: 0.01 # 1% From 8826a0245aaabbc17017437bd54bdd6bbbea38b5 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Fri, 17 Apr 2026 14:13:24 -0700 Subject: [PATCH 2/5] added cpln-common template for tagging --- manticore/versions/2.0.0/Chart.yaml | 7 ++++- .../versions/2.0.0/templates/_helpers.tpl | 28 ++----------------- 2 files changed, 8 insertions(+), 27 deletions(-) diff --git a/manticore/versions/2.0.0/Chart.yaml b/manticore/versions/2.0.0/Chart.yaml index a7526e95..08f47032 100644 --- a/manticore/versions/2.0.0/Chart.yaml +++ b/manticore/versions/2.0.0/Chart.yaml @@ -10,4 +10,9 @@ annotations: created: "2026-01-05" lastModified: "2026-04-17" category: "search" - createsGvc: false \ No newline at end of file + createsGvc: false + +dependencies: + - name: cpln-common + version: 1.0.0 + repository: "oci://ghcr.io/controlplane-com/templates" \ No newline at end of file diff --git a/manticore/versions/2.0.0/templates/_helpers.tpl b/manticore/versions/2.0.0/templates/_helpers.tpl index d378f13d..c0326b1d 100644 --- a/manticore/versions/2.0.0/templates/_helpers.tpl +++ b/manticore/versions/2.0.0/templates/_helpers.tpl @@ -235,32 +235,8 @@ Parses duration strings like "5m", "1h", "30s" {{/* Labeling */}} {{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "manticore.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels +Common labels - delegated to cpln-common */}} {{- define "manticore.tags" -}} -helm.sh/chart: {{ include "manticore.chart" . }} -{{ include "manticore.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.cpln.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.cpln.io/managed-by: {{ .Release.Service }} -cpln/marketplace: "true" -cpln/marketplace-template: manticore -cpln/marketplace-template-version: {{ .Chart.Version }} -cpln/marketplace-gvc: {{ .Values.global.cpln.gvc }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "manticore.selectorLabels" -}} -app.cpln.io/name: {{ .Release.Name }} -app.cpln.io/instance: {{ .Release.Name }} +{{- include "cpln-common.tags" . }} {{- end }} \ No newline at end of file From 183ac87a0105f313e7c2c4d65d76f0abd992532c Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Fri, 17 Apr 2026 18:37:13 -0700 Subject: [PATCH 3/5] added buddy for metrics, specified plugin_dir in config --- .../2.0.0/charts/cpln-common-1.0.0.tgz | Bin 0 -> 680 bytes .../versions/2.0.0/templates/secret.yaml | 6 ++++- manticore/versions/2.0.0/values.yaml | 21 +++++++++--------- 3 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 manticore/versions/2.0.0/charts/cpln-common-1.0.0.tgz diff --git a/manticore/versions/2.0.0/charts/cpln-common-1.0.0.tgz b/manticore/versions/2.0.0/charts/cpln-common-1.0.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..c41e09051d41e5bd4feeabac39714835119f385c GIT binary patch literal 680 zcmV;Z0$2SXiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PIyykJCO7?Kz)fWN9z^Q(X67eo}6z2RHyiRNCvxO(uzjV+Y$Q zRNejV$oW&UX_rM}#RBSF;yBNq_g*~Xq>?I3bXjUOT^HOqy62^wJZmtgGsqpLF_1Z%M{COg4Xm|tMoai0xk4zp`Gl^LJV9_+R?-s)6fZG`!ATC> zzQXSkc74(rWt0UkZTTM+!}>ouI=wAWR%k4B)id;hC+dF|MaxnBN9=b9wq%YG-x(C970Gj6AFP0 z&KPZw1i}td8KGPTHP5^TyLQ@p;TxH0@c6tp6Ras;d7ZBdy0S z>!qH6@8EN--B|W;eaRKiF%7r-hi+}q>_pP6^w8> "${RUNTIME_CONFIG}" echo " listen = 127.0.0.1:9312" >> "${RUNTIME_CONFIG}" echo " listen = ${MY_IP}:9312" >> "${RUNTIME_CONFIG}" - echo " listen = ${MY_IP}:9313-9320:replication" >> "${RUNTIME_CONFIG}" + echo " listen = 0.0.0.0:9313-9320:replication" >> "${RUNTIME_CONFIG}" echo " node_address = ${MY_FQDN}" >> "${RUNTIME_CONFIG}" echo " data_dir = /var/lib/manticore" >> "${RUNTIME_CONFIG}" echo " binlog_path = /var/lib/manticore/binlog" >> "${RUNTIME_CONFIG}" @@ -197,6 +197,10 @@ data: echo " seamless_rotate = 1" >> "${RUNTIME_CONFIG}" echo " preopen_tables = 1" >> "${RUNTIME_CONFIG}" echo " server_id = ${REPLICA_INDEX}" >> "${RUNTIME_CONFIG}" + echo " buddy_path = /usr/share/manticore/modules/manticore-buddy/bin/manticore-buddy" >> "${RUNTIME_CONFIG}" + echo "}" >> "${RUNTIME_CONFIG}" + echo "common {" >> "${RUNTIME_CONFIG}" + echo " plugin_dir = /usr/share/manticore/modules" >> "${RUNTIME_CONFIG}" echo "}" >> "${RUNTIME_CONFIG}" echo "Runtime config generated:" diff --git a/manticore/versions/2.0.0/values.yaml b/manticore/versions/2.0.0/values.yaml index 12b7e1f2..f40d8b7e 100644 --- a/manticore/versions/2.0.0/values.yaml +++ b/manticore/versions/2.0.0/values.yaml @@ -3,11 +3,11 @@ # ============================================================================= # See README.md for Cloud Account and IAM policy setup instructions. buckets: - cloudAccountName: my-cloud-account # Name of your configured Cloud Account + cloudAccountName: my-s3-cloudaccount # Name of your configured Cloud Account awsPolicyRefs: # IAM policies for S3 access - - aws::AmazonS3ReadOnlyAccess # Note: if using a custom policy, omit the aws:: prefix as this is only for AWS managed policies + - manticore-test-policy # Note: if using a custom policy, omit the aws:: prefix as this is only for AWS managed policies awsRegion: us-east-1 # Region of your S3 bucket - sourceBucket: my-bucket # S3 bucket containing files to import + sourceBucket: cpln-manticore-test # S3 bucket containing files to import # ============================================================================= # Tables Configuration @@ -17,12 +17,13 @@ buckets: tables: - name: addresses csvPath: - - manticore-imports/addresses/data.csv + - small_addresses_1.csv + - small_addresses_2.csv config: haStrategy: noerrors agentRetryCount: 3 clusterMain: false - segmentCount: 1 + segmentCount: 2 charsetTable: non_cont memLimit: 2G hasHeader: true @@ -88,7 +89,7 @@ manticore: # Orchestrator Configuration # ============================================================================= orchestrator: - version: v6.0.0 + version: v6.0.3 image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-api logLevel: debug # debug, info, warn, error resources: @@ -106,7 +107,7 @@ orchestrator: # Orchestrator API api: - version: v6.0.0 + version: v6.0.3 image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-api logLevel: debug importPollInterval: 30s @@ -122,7 +123,7 @@ orchestrator: # Agent sidecar agent: - version: v6.0.0 + version: v6.0.3 image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-agent # REQUIRED: Generate with `openssl rand -base64 32` token: "6Gl5uO9KkKAh1u+ymoBW98WCtjTFpljuhpLdKb+tNAA=" @@ -140,7 +141,7 @@ orchestrator: # Web UI - See README.md "Authentication" section for security notes ui: - version: v6.0.0 + version: v6.0.3 image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-ui resources: cpu: 0.25 @@ -155,7 +156,7 @@ orchestrator: # Backup Configuration (Optional) - Cron job runs logical backup on delta table to S3 bucket. backup: enabled: false - version: v6.0.0 + version: v6.0.3 image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-backup cloudAccountName: my-backup-cloud-account s3Bucket: my-backup-bucket # S3 bucket for backups From 1499d9eac0473c308bf6a6eacf9ccc3d0d45af95 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Mon, 20 Apr 2026 15:41:01 -0700 Subject: [PATCH 4/5] updated replication listener and ports --- manticore/versions/2.0.0/templates/secret.yaml | 3 ++- .../versions/2.0.0/templates/workload.yaml | 18 +++--------------- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/manticore/versions/2.0.0/templates/secret.yaml b/manticore/versions/2.0.0/templates/secret.yaml index f479d1c2..f2cd4261 100644 --- a/manticore/versions/2.0.0/templates/secret.yaml +++ b/manticore/versions/2.0.0/templates/secret.yaml @@ -187,7 +187,8 @@ data: echo " listen = ${MY_IP}:9308:http" >> "${RUNTIME_CONFIG}" echo " listen = 127.0.0.1:9312" >> "${RUNTIME_CONFIG}" echo " listen = ${MY_IP}:9312" >> "${RUNTIME_CONFIG}" - echo " listen = 0.0.0.0:9313-9320:replication" >> "${RUNTIME_CONFIG}" + echo " listen = 0.0.0.0:9322-9323:replication" >> "${RUNTIME_CONFIG}" + echo " node_address = ${MY_FQDN}" >> "${RUNTIME_CONFIG}" echo " data_dir = /var/lib/manticore" >> "${RUNTIME_CONFIG}" echo " binlog_path = /var/lib/manticore/binlog" >> "${RUNTIME_CONFIG}" diff --git a/manticore/versions/2.0.0/templates/workload.yaml b/manticore/versions/2.0.0/templates/workload.yaml index 8acc6b6d..b565ecd1 100644 --- a/manticore/versions/2.0.0/templates/workload.yaml +++ b/manticore/versions/2.0.0/templates/workload.yaml @@ -27,22 +27,10 @@ spec: protocol: http - number: 9312 protocol: tcp - # Galera replication ports - - number: 9313 + # Galera gcomm + IST ports + - number: 9322 protocol: tcp - - number: 9314 - protocol: tcp - - number: 9315 - protocol: tcp - - number: 9316 - protocol: tcp - - number: 9317 - protocol: tcp - - number: 9318 - protocol: tcp - - number: 9319 - protocol: tcp - - number: 9320 + - number: 9323 protocol: tcp readinessProbe: failureThreshold: 6 From bbf24634275f1c1602b19aef83ad39477cd1f049 Mon Sep 17 00:00:00 2001 From: Jacob Cox Date: Mon, 20 Apr 2026 16:26:28 -0700 Subject: [PATCH 5/5] upgraded image versions, added buddy plugin for metrics --- .../2.0.0/charts/cpln-common-1.0.0.tgz | Bin 680 -> 0 bytes .../versions/2.0.0/templates/secret.yaml | 6 ++++- manticore/versions/2.0.0/values.yaml | 21 +++++++++--------- 3 files changed, 15 insertions(+), 12 deletions(-) delete mode 100644 manticore/versions/2.0.0/charts/cpln-common-1.0.0.tgz diff --git a/manticore/versions/2.0.0/charts/cpln-common-1.0.0.tgz b/manticore/versions/2.0.0/charts/cpln-common-1.0.0.tgz deleted file mode 100644 index c41e09051d41e5bd4feeabac39714835119f385c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 680 zcmV;Z0$2SXiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PIyykJCO7?Kz)fWN9z^Q(X67eo}6z2RHyiRNCvxO(uzjV+Y$Q zRNejV$oW&UX_rM}#RBSF;yBNq_g*~Xq>?I3bXjUOT^HOqy62^wJZmtgGsqpLF_1Z%M{COg4Xm|tMoai0xk4zp`Gl^LJV9_+R?-s)6fZG`!ATC> zzQXSkc74(rWt0UkZTTM+!}>ouI=wAWR%k4B)id;hC+dF|MaxnBN9=b9wq%YG-x(C970Gj6AFP0 z&KPZw1i}td8KGPTHP5^TyLQ@p;TxH0@c6tp6Ras;d7ZBdy0S z>!qH6@8EN--B|W;eaRKiF%7r-hi+}q>_pP6^w8> "${RUNTIME_CONFIG}" echo " data_dir = /var/lib/manticore" >> "${RUNTIME_CONFIG}" echo " binlog_path = /var/lib/manticore/binlog" >> "${RUNTIME_CONFIG}" - echo " log = /dev/stdout" >> "${RUNTIME_CONFIG}" + echo " log = /var/lib/manticore/searchd.log" >> "${RUNTIME_CONFIG}" echo " query_log = /dev/stdout" >> "${RUNTIME_CONFIG}" echo " pid_file = /var/run/searchd.pid" >> "${RUNTIME_CONFIG}" echo " seamless_rotate = 1" >> "${RUNTIME_CONFIG}" @@ -208,12 +208,16 @@ data: cat "${RUNTIME_CONFIG}" echo "" + chmod +x /usr/share/manticore/modules/manticore-buddy/bin/manticore-buddy + echo "============================================" echo "Starting searchd..." echo "============================================" echo "" echo "Launching searchd in background..." + touch /var/lib/manticore/searchd.log + tail -f /var/lib/manticore/searchd.log & searchd --config "${RUNTIME_CONFIG}" --nodetach 2>&1 & SEARCHD_PID="$!" echo "searchd launched with PID: ${SEARCHD_PID}" diff --git a/manticore/versions/2.0.0/values.yaml b/manticore/versions/2.0.0/values.yaml index f40d8b7e..883cafcd 100644 --- a/manticore/versions/2.0.0/values.yaml +++ b/manticore/versions/2.0.0/values.yaml @@ -3,11 +3,11 @@ # ============================================================================= # See README.md for Cloud Account and IAM policy setup instructions. buckets: - cloudAccountName: my-s3-cloudaccount # Name of your configured Cloud Account + cloudAccountName: my-cloudaccount # Name of your configured Cloud Account awsPolicyRefs: # IAM policies for S3 access - - manticore-test-policy # Note: if using a custom policy, omit the aws:: prefix as this is only for AWS managed policies + - my-manticore-policy # Note: if using a custom policy, omit the aws:: prefix as this is only for AWS managed policies awsRegion: us-east-1 # Region of your S3 bucket - sourceBucket: cpln-manticore-test # S3 bucket containing files to import + sourceBucket: my-manticore-bucket # S3 bucket containing files to import # ============================================================================= # Tables Configuration @@ -17,13 +17,12 @@ buckets: tables: - name: addresses csvPath: - - small_addresses_1.csv - - small_addresses_2.csv + - imports/addresses.csv config: haStrategy: noerrors agentRetryCount: 3 clusterMain: false - segmentCount: 2 + segmentCount: 1 charsetTable: non_cont memLimit: 2G hasHeader: true @@ -89,7 +88,7 @@ manticore: # Orchestrator Configuration # ============================================================================= orchestrator: - version: v6.0.3 + version: v6.0.4 image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-api logLevel: debug # debug, info, warn, error resources: @@ -107,7 +106,7 @@ orchestrator: # Orchestrator API api: - version: v6.0.3 + version: v6.0.4 image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-api logLevel: debug importPollInterval: 30s @@ -123,7 +122,7 @@ orchestrator: # Agent sidecar agent: - version: v6.0.3 + version: v6.0.4 image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-agent # REQUIRED: Generate with `openssl rand -base64 32` token: "6Gl5uO9KkKAh1u+ymoBW98WCtjTFpljuhpLdKb+tNAA=" @@ -141,7 +140,7 @@ orchestrator: # Web UI - See README.md "Authentication" section for security notes ui: - version: v6.0.3 + version: v6.0.4 image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-ui resources: cpu: 0.25 @@ -156,7 +155,7 @@ orchestrator: # Backup Configuration (Optional) - Cron job runs logical backup on delta table to S3 bucket. backup: enabled: false - version: v6.0.3 + version: v6.0.4 image: ghcr.io/controlplane-com/manticore-orchestrator/manticore-cpln-backup cloudAccountName: my-backup-cloud-account s3Bucket: my-backup-bucket # S3 bucket for backups