fix: persist refreshed viya tokens in deploy flow to avoid stale-toke…

docs/diagrams/auth-token-refresh.md

@@ -0,0 +1,73 @@
+# Access/refresh token & auth config logic (SAS Viya)
+
+Scope: `src/utils/auth.ts`, `src/utils/config.ts`. Two entry points resolve
+credentials for a `Target`; they differ in one critical way (persistence). Read the
+"Critical invariant" section before touching either.
+
+## Entry points comparison
+
+```mermaid
+flowchart TD
+    subgraph getAuthConfig["getAuthConfig(target) — config.ts:600 — use this from any command"]
+        A1["read access_token: target.authConfig.access_token\nelse overrideEnvVariables(target.name) + process.env.ACCESS_TOKEN"] --> A2["read client/secret/refresh_token\ntarget.authConfig.* else process.env.CLIENT/SECRET/REFRESH_TOKEN"]
+        A2 --> A3{"client missing?"}
+        A3 -->|yes| A3E["throw 'Client ID was not found...'"]
+        A3 -->|no| A4{"secret missing?"}
+        A4 -->|yes, not Sasjs server| A4E["throw 'Client secret was not found...'"]
+        A4 -->|no| A5{"isAccessTokenExpiring(access_token)\nauth.ts:37 — exp - now <= 1hr"}
+        A5 -->|no| A6["return {access_token, refresh_token, client, secret}"]
+        A5 -->|yes| A7{"isRefreshTokenExpiring(refresh_token)\nauth.ts:54 — exp - now <= 30s"}
+        A7 -->|yes: refresh token dead too| A8["getNewAccessToken(...) auth.ts:79\nINTERACTIVE: prompts user for auth code via browser URL"]
+        A7 -->|no: refresh token still valid| A9["refreshTokens(sasjs, client, secret, refresh_token) auth.ts:64\ncalls sasjsInstance.refreshTokens → NEW access_token + NEW refresh_token"]
+        A8 --> A10["saveTokens(target.name, client, secret, access_token, refresh_token) config.ts:684\n★ persists new pair to .env.{target} (local) or ~/.sasjsrc authConfig (global)"]
+        A9 --> A10
+        A10 --> A6
+    end
+```
+
+```mermaid
+flowchart TD
+    subgraph getAccessToken["getAccessToken(target) — config.ts:772 — test-cleanup only"]
+        B1["read accessToken: target.authConfig.access_token\nelse overrideEnvVariables(target.name) + process.env.ACCESS_TOKEN"] --> B2{"accessToken still empty?"}
+        B2 -->|yes| B2E["throw 'A valid access token was not found...'"]
+        B2 -->|no| B3{"checkIfExpiring && isAccessTokenExpiring(accessToken)"}
+        B3 -->|no| B6["return accessToken"]
+        B3 -->|yes| B4["read client/secret/refresh_token from\ntarget.authConfig.* or process.env.*\n(throws if client/secret missing, same messages as getAuthConfig)"]
+        B4 --> B5{"isRefreshTokenExpiring(refresh_token)"}
+        B5 -->|yes| B7["getNewAccessToken(...) — interactive"]
+        B5 -->|no| B8["refreshTokens(...) → NEW access_token + NEW refresh_token"]
+        B7 --> B9["accessToken = tokens.access_token\nnew refresh_token is not written to disk"]
+        B8 --> B9
+        B9 --> B6
+    end
+```
+
+## Callers
+
+```mermaid
+flowchart LR
+    deploy["deployToSasViyaWithServicePack.ts"] --> getAuthConfig
+    folder["folderCommand.ts"] --> getAuthConfig
+    job["job/internal/execute/viya.ts"] --> getAuthConfig
+    request["request.ts / flow / context"] --> getAuthConfig
+    testCleanup["utils/test.ts: removeTestServerFolder()"] --> getAccessToken
+```
+
+## Critical invariant
+
+- **Any code path that can be invoked again in a later, separate process** (i.e. any
+  real CLI command) **must persist a refreshed token pair** — use `getAuthConfig()`.
+  SAS Viya issues single-use, rotating refresh tokens: every `refreshTokens()` call
+  invalidates the refresh token it was given and returns a new one, so the new pair
+  has to be written back to disk or the next invocation has nothing valid to use.
+- **`getAccessToken()` never persists.** Safe only for same-process, one-shot use that
+  nothing else depends on afterward — currently only `removeTestServerFolder()` in
+  `src/utils/test.ts:94` (test cleanup).
+- Expiry thresholds: access token refreshed if `exp - now <= 3600s` (1hr);
+  refresh token considered dead if `exp - now <= 30s`.
+- If the refresh token is dead, both entry points fall back to `getNewAccessToken()`,
+  which is **interactive** (prints a URL, prompts for an auth code) — unsuitable for
+  unattended/CI use.
+- Persistence target depends on where the target lives: local `sasjsconfig.json` →
+  writes `.env.{targetName}`; global `~/.sasjsrc` → writes into that target's
+  `authConfig` object directly (`saveTokens`, config.ts:684).

docs/diagrams/sasjs-compile.md

@@ -0,0 +1,80 @@
+# `sasjs compile` (`sasjs c`) logic
+
+Scope: `src/commands/compile/compile.ts`, `src/commands/compile/internal/*`. Entry
+point is `compile(target, forceCompile)` at `compile.ts:40`.
+
+## Top-level flow
+
+```mermaid
+flowchart TD
+    Start(["compile(target, forceCompile) — compile.ts:40"]) --> Check["checkCompileStatus(target, ['tests']) — checkCompileStatus.ts:10\ncompares sasjsbuild/services & sasjsbuild/jobs against source folders via compareFolders()"]
+    Check --> Skip{"result.compiled && !forceCompile?"}
+    Skip -->|yes| Log["log 'Skipping compilation.' + reason, return"]
+    Skip -->|no| Recreate["recreateBuildFolder() — delete + recreate sasjsbuild/"]
+    Recreate --> Copy["copyFilesToBuildFolder(target) — compile.ts:109\ncopy() each service/job source folder verbatim into sasjsbuild/services|jobs"]
+    Copy --> CJST["compileJobsServicesTests(target, compileTree) — compile.ts:141\n(see subgraph below)"]
+    CJST --> MacroTests["for each macroFolder: copyTestMacroFiles()\nthen compileFile() on every *.test.sas found under sasjsbuild/tests/macros"]
+    MacroTests --> SaveTree["compileTree.saveTree() → {target}_compileTree.json"]
+    SaveTree --> Flow["compileTestFlow(target) — errors logged, not thrown\n(see subgraph below)"]
+    Flow --> Web{"streamConfig.streamWeb enabled?"}
+    Web -->|yes| WebGen["createWebAppServices(target) — compiles streaming web app services"]
+    Web -->|no| Sync
+    WebGen --> Sync["syncFolder(target): copySyncFolder() for config.syncFolder and target.syncFolder"]
+    Sync --> End(["done"])
+```
+
+## `compileJobsServicesTests` — per-file dispatch
+
+```mermaid
+flowchart TD
+    A["compileJobsServicesTests(target, compileTree) — compile.ts:141"] --> B["getAllFolders(target, Service) / getAllFolders(target, Job)\ngetMacroFolders(target) / getProgramFolders(target)\ngetTestSetUp(target) / getTestTearDown(target) — raw testConfig.testSetUp/testTearDown string"]
+    B --> C{"testSetUp configured?"}
+    C -->|yes| D["compileTestFile(target, testSetUp, '', saveToRoot=true, removeOriginalFile=false, compileTree)\n→ written flat as sasjsbuild/tests/{basename}"]
+    C -->|no| E
+    D --> E{"testTearDown configured?"}
+    E -->|yes| F["compileTestFile(..., testTearDown, saveToRoot=true, ...) → sasjsbuild/tests/{basename}"]
+    E -->|no| G
+    F --> G["for each serviceFolder: compileServiceFolder(...)\nfor each jobFolder: compileJobFolder(...)"]
+    G --> H["per file in folder (incl. one level of subfolders):\nisTestFile(fileName)?"]
+    H -->|yes, *.test.sas| I["compileTestFile(target, filePath, '', saveToRoot=false, ..., compileTree)\n→ preserves path fragment after buildDestinationFolder name,\ninside sasjsbuild/tests/{services|jobs}/..."]
+    H -->|no| J["compileFile(target, filePath, macroFolders, programFolders, ..., fileType, sourceFolder)\n→ loadDependencies() resolves %macro/[include] headers, writes resolved SAS in place"]
+```
+
+## `compileTestFlow` — building `testFlow.json` + coverage
+
+```mermaid
+flowchart TD
+    A["compileTestFlow(target) — compileTestFile.ts:98"] --> B{"sasjsbuild/tests folder exists?"}
+    B -->|no| Z["return undefined"]
+    B -->|yes| C["testFiles = listFilesAndSubFoldersInFolder(buildDestinationTestFolder)\nprefixed with 'tests/'"]
+    C --> D{"testSetUp / testTearDown configured\n(target.testConfig or root sasjsconfig.json)?"}
+    D --> E["match by basename against testFiles;\nremove matched entry from testFiles list,\nset testFlow.testSetUp / testFlow.testTearDown = 'tests/{basename}'"]
+    E --> F["testFlow.tests = remaining testFiles (posix-joined)"]
+    F --> G["printTestCoverage(testFlow, buildDestinationFolder, target)"]
+    G --> H["collectCoverage() over sasjsbuild/services, sasjsbuild/jobs,\nand each macro folder (excluding *.test.sas)"]
+    H --> I["for each coverable file, check whether a matching entry\nexists in testFlow.tests (by stripping .test[.N].sas suffix)\n→ covered vs notCovered; unmatched testFlow entries → 'standalone'"]
+    I --> J["log coverage table + per-type percentage\nwrite testFlow.json = { tests, testSetUp?, testTearDown? } to buildDestinationFolder"]
+```
+
+## Key facts
+
+- `checkCompileStatus` can make `compile()` a no-op: it diffs source vs already-compiled
+  output folder-by-folder (`compareFolders`) and skips work entirely unless
+  `forceCompile` is passed or something actually changed.
+- `testSetUp`/`testTearDown` are compiled once, flattened to the root of
+  `sasjsbuild/tests/` (`saveToRoot = true`); all other `*.test.sas` files found under
+  service/job folders are compiled with their relative path preserved under
+  `sasjsbuild/tests/{services|jobs}/...` (`saveToRoot = false`). Both paths go through
+  `getTestFileDestinationFragment(filePath, buildDestinationFolderName, saveToRoot)`
+  in `compileTestFile.ts`.
+- Macro test files (`*.test.sas` under any macro folder) are handled separately via
+  `copyTestMacroFiles()` (copy into `sasjsbuild/tests/macros`) + a second `compileFile()`
+  pass to resolve their dependencies - this happens after `compileJobsServicesTests`,
+  not inside it.
+- `compileTestFlow` only ever *reads* whatever already landed in
+  `sasjsbuild/tests` - it doesn't compile anything itself, it just assembles
+  `testFlow.json` and prints/writes coverage.
+- Dependency resolution (`%macro`/program includes) for every non-test file goes
+  through `loadDependencies()` → `loadDependenciesFile()` (from `@sasjs/utils`), which
+  is also memoized per-file via the `compileTree` (`{target}_compileTree.json`) to
+  avoid recomputing dependencies across compiles.

src/commands/deploy/spec/cbd.spec.server.viya.ts

@@ -140,15 +140,15 @@ describe('sasjs cbd with Viya', () => {
     })
 
     it(`should error when an access token is not provided`, async () => {
-      jest.spyOn(configUtils, 'getAccessToken').mockImplementation(() => {
+      jest.spyOn(configUtils, 'getAuthConfig').mockImplementation(() => {
         return Promise.reject('Token error')
       })
 
       await build(target)
 
       await expect(deploy(target, true)).rejects.toThrow(
         new Error(
-          `Deployment failed. Request is not authenticated.\nPlease add the following variables to your .env.${target.name} file:\nCLIENT, SECRET, ACCESS_TOKEN, REFRESH_TOKEN`
+          `Deployment failed. Request is not authenticated.\nToken error`
         )
       )
 

src/commands/shared/deployToSasViyaWithServicePack.ts

@@ -1,5 +1,5 @@
 import { FileTree, MemberType, readFile, Target } from '@sasjs/utils'
-import { getAccessToken, getSASjs } from '../../utils'
+import { getAuthConfig, getSASjs } from '../../utils'
 
 export async function deployToSasViyaWithServicePack(
   jsonFilePath: string,
@@ -19,15 +19,16 @@ export async function deployToSasViyaWithServicePack(
 
   populateCodeInServicePack(jsonObject)
 
-  const access_token: string = await getAccessToken(target).catch((e) => '')
-
-  if (!access_token) {
+  // getAuthConfig (not getAccessToken) is required here: it persists a
+  // refreshed access/refresh token pair back to disk, which matters because
+  // Viya refresh tokens are single-use/rotating - see getAuthConfig's doc.
+  // The original error is forwarded rather than replaced, since it already
+  // describes the real cause (missing client/secret, or a rejected refresh).
+  const { access_token } = await getAuthConfig(target).catch((err) => {
     throw new Error(
-      `Deployment failed. Request is not authenticated.\nPlease add the following variables to your .env${
-        isLocal ? `.${target.name}` : ''
-      } file:\nCLIENT, SECRET, ACCESS_TOKEN, REFRESH_TOKEN`
+      `Deployment failed. Request is not authenticated.\n${err?.message || err}`
     )
-  }
+  })
 
   const sasjs = getSASjs(target)
 

src/commands/shared/spec/deployToSasViyaWithServicePack.spec.ts

@@ -0,0 +1,78 @@
+import path from 'path'
+import os from 'os'
+import {
+  AuthConfig,
+  ServerType,
+  Target,
+  createFile,
+  deleteFile
+} from '@sasjs/utils'
+import * as configUtils from '../../../utils/config'
+import { deployToSasViyaWithServicePack } from '../deployToSasViyaWithServicePack'
+
+const target = new Target({
+  name: 'test',
+  appLoc: '/Public/test/',
+  serverType: ServerType.SasViya,
+  serverUrl: 'https://server.com',
+  contextName: 'test context'
+})
+
+const mockAuthConfig: AuthConfig = {
+  client: 'cl13nt',
+  secret: '53cr3t',
+  access_token: 'acc355',
+  refresh_token: 'r3fr35h'
+}
+
+describe('deployToSasViyaWithServicePack', () => {
+  let jsonFilePath: string
+  let deployServicePack: jest.Mock
+
+  beforeEach(async () => {
+    jsonFilePath = path.join(os.tmpdir(), `servicepack-${Date.now()}.json`)
+    await createFile(jsonFilePath, JSON.stringify({ members: [] }))
+
+    deployServicePack = jest.fn().mockResolvedValue({})
+    jest
+      .spyOn(configUtils, 'getSASjs')
+      .mockImplementation(() => ({ deployServicePack } as any))
+  })
+
+  afterEach(async () => {
+    await deleteFile(jsonFilePath)
+    jest.restoreAllMocks()
+  })
+
+  it('refreshes and persists the access token via getAuthConfig instead of the non-persisting getAccessToken', async () => {
+    const getAuthConfigSpy = jest
+      .spyOn(configUtils, 'getAuthConfig')
+      .mockImplementation(() => Promise.resolve(mockAuthConfig))
+
+    await expect(
+      deployToSasViyaWithServicePack(jsonFilePath, target, true, false)
+    ).toResolve()
+
+    expect(getAuthConfigSpy).toHaveBeenCalledWith(target)
+    expect(deployServicePack).toHaveBeenCalledWith(
+      { members: [] },
+      undefined,
+      undefined,
+      mockAuthConfig.access_token,
+      false
+    )
+  })
+
+  it('surfaces the real auth/refresh error instead of a generic "add these variables" message when credentials already exist', async () => {
+    const realError = new Error(
+      'invalid_grant: refresh token has already been used'
+    )
+    jest.spyOn(configUtils, 'getAuthConfig').mockImplementation(() => {
+      return Promise.reject(realError)
+    })
+
+    await expect(
+      deployToSasViyaWithServicePack(jsonFilePath, target, true, false)
+    ).rejects.toThrow(/invalid_grant: refresh token has already been used/)
+  })
+})

src/utils/auth.ts

@@ -61,6 +61,14 @@ export function isRefreshTokenExpiring(token?: string): boolean {
   return timeToLive <= 30 // 30 seconds
 }
 
+/**
+ * Exchanges a refresh token for a new access/refresh token pair.
+ * SAS Viya's refresh tokens are single-use and rotate on every call: the
+ * `refresh_token` returned here supersedes the one passed in, which becomes
+ * invalid immediately. Callers must persist the returned pair (see
+ * `saveTokens` in config.ts) or a later refresh attempt with the old token
+ * will be rejected by the server.
+ */
 export async function refreshTokens(
   sasjsInstance: SASjs,
   clientId: string,

src/utils/config.ts

@@ -593,7 +593,18 @@ export async function getProjectRoot() {
 }
 
 /**
- * Gets the auth config for the specified target.
+ * Gets the auth config for the specified target, refreshing the access token
+ * (via the refresh token) if it is expiring.
+ *
+ * Unlike {@link getAccessToken}, this persists any newly issued access/refresh
+ * token pair via {@link saveTokens}. That persistence is not optional: SAS Viya
+ * issues rotating, single-use refresh tokens, so once a refresh token has been
+ * used to mint a new pair, the old one is void. If the new pair isn't written
+ * back to `.env.{target}` / the global config, the *next* CLI invocation reads
+ * the now-stale refresh token from disk and fails to authenticate - which is
+ * indistinguishable, from the user's point of view, from never having
+ * refreshed at all. This is why every command that needs an authenticated
+ * request should call `getAuthConfig`, not `getAccessToken`.
  * @param {Target} target - the target to get an access token for.
  * @returns {AuthConfig} - an object containing an access token, refresh token, client ID and secret.
  */
@@ -681,6 +692,13 @@ export async function getAuthConfig(target: Target): Promise<AuthConfig> {
   }
 }
 
+/**
+ * Persists an access/refresh token pair to wherever the target's credentials
+ * live: `.env.{targetName}` for a local target, or `authConfig` in the global
+ * `~/.sasjsrc` otherwise. Called by {@link getAuthConfig} after every refresh
+ * so that a rotated (single-use) refresh token from SAS Viya is never reused -
+ * reusing it on a subsequent CLI invocation would be rejected by the server.
+ */
 export const saveTokens = async (
   targetName: string,
   client: string,
@@ -766,6 +784,16 @@ export function getAuthConfigSAS9(target: Target): AuthConfigSas9 {
  * If a refresh token is unavailable, we will use the client ID and secret
  * to obtain a new access token. Manual intervention is required in this case
  * to navigate to the URL shown and type in an authorization code.
+ *
+ * IMPORTANT: unlike {@link getAuthConfig}, this does NOT persist a refreshed
+ * access/refresh token pair back to disk - the new tokens only live for the
+ * lifetime of this process. Since SAS Viya refresh tokens are single-use and
+ * rotate on every refresh, calling this from a command whose CLI invocation
+ * ends shortly after (leaving the stale refresh token on disk) will cause the
+ * *next* invocation to fail to authenticate. Only use this where that's
+ * acceptable - e.g. one-shot, same-process cleanup calls such as
+ * `removeTestServerFolder` in test support code. Any user-facing command
+ * should use `getAuthConfig` instead.
  * @param {object} target - the target to get an access token for.
  * @param {string} checkIfExpiring - flag that indicates whether to do an expiry check.
  */

src/utils/test.ts

@@ -91,6 +91,9 @@ export const removeTestServerFolder = async (
 
   const sasjs = getSASjs(target)
 
+  // getAccessToken is fine here (unlike in a real command) since this is a
+  // one-shot, same-process cleanup call - no later invocation depends on the
+  // refresh token this leaves on disk. See getAccessToken's doc in config.ts.
   const accessToken = await getAccessToken(target)
 
   await deleteServerFolder(folderPath, sasjs, accessToken)