Skip to content

Use exec when launching Gradle #3897

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
sjchmiela wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Use exec when launching Gradle #3897

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 10 additions & 63 deletions packages/build-tools/src/android/gradle.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
import { Android, Env, Job, Platform } from '@expo/eas-build-job';
import { bunyan } from '@expo/logger';
import spawn, { SpawnPromise, SpawnResult } from '@expo/turtle-spawn';
import assert from 'assert';
import fs from 'fs-extra';
import path from 'path';

import { BuildContext } from '../context';
import { getParentAndDescendantProcessPidsAsync } from '../utils/processes';
import { runGradleCommand as runGradleCommandInternal } from '../utils/gradle';

export async function ensureLFLineEndingsInGradlewScript<TJob extends Job>(
ctx: BuildContext<TJob>
Expand All @@ -28,67 +26,16 @@ export async function runGradleCommand(
extraEnv,
}: { logger: bunyan; gradleCommand: string; androidDir: string; extraEnv?: Env }
): Promise<void> {
logger.info(`Running 'gradlew ${gradleCommand}' in ${androidDir}`);
await fs.chmod(path.join(androidDir, 'gradlew'), 0o755);
const verboseFlag = ctx.env['EAS_VERBOSE'] === '1' ? '--info' : '';

const spawnPromise = spawn(
'bash',
['-c', `./gradlew ${gradleCommand} --profile ${verboseFlag}`],
{
cwd: androidDir,
logger,
lineTransformer: (line?: string) => {
if (!line || /^\.+$/.exec(line)) {
return null;
} else {
return line;
}
},
env: {
...ctx.env,
...extraEnv,
...resolveVersionOverridesEnvs(ctx),
LC_ALL: 'C.UTF-8',
},
}
);
if (ctx.env.EAS_BUILD_RUNNER === 'eas-build' && process.platform === 'linux') {
adjustOOMScore(spawnPromise, logger);
}

await spawnPromise;
}

/**
* OOM Killer sometimes kills worker server while build is exceeding memory limits.
* `oom_score_adj` is a value between -1000 and 1000 and it defaults to 0.
* It defines which process is more likely to get killed (higher value more likely).
*
* This function sets oom_score_adj for Gradle process and all its child processes.
*/
function adjustOOMScore(spawnPromise: SpawnPromise<SpawnResult>, logger: bunyan): void {
setTimeout(
async () => {
try {
assert(spawnPromise.child.pid);
const pids = await getParentAndDescendantProcessPidsAsync(spawnPromise.child.pid);
await Promise.all(
pids.map(async (pid: number) => {
// Value 800 is just a guess here. It's probably higher than most other
// process. I didn't want to set it any higher, because I'm not sure if OOM Killer
// can start killing processes when there is still enough memory left.
const oomScoreOverride = 800;
await fs.writeFile(`/proc/${pid}/oom_score_adj`, `${oomScoreOverride}\n`);
})
);
} catch (err: any) {
logger.debug({ err, stderr: err?.stderr }, 'Failed to override oom_score_adj');
}
await runGradleCommandInternal({
androidDir,
env: {
...ctx.env,
...extraEnv,
...resolveVersionOverridesEnvs(ctx),
},
// Wait 20 seconds to make sure all child processes are started
20000
);
gradleCommand,
logger,
});
}

// Version envs should be set at the beginning of the build, but when building
Expand Down
113 changes: 1 addition & 112 deletions packages/build-tools/src/steps/utils/android/gradle.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,117 +1,6 @@
import { Android } from '@expo/eas-build-job';
import { bunyan } from '@expo/logger';
import { BuildStepEnv } from '@expo/steps';
import spawn, { SpawnPromise, SpawnResult } from '@expo/turtle-spawn';
import assert from 'assert';
import fs from 'fs-extra';
import path from 'path';

export async function runGradleCommand({
logger,
gradleCommand,
androidDir,
env,
extraEnv,
}: {
logger: bunyan;
gradleCommand: string;
androidDir: string;
env: BuildStepEnv;
extraEnv?: BuildStepEnv;
}): Promise<void> {
const verboseFlag = env['EAS_VERBOSE'] === '1' ? '--info' : '';

logger.info(`Running 'gradlew ${gradleCommand} ${verboseFlag}' in ${androidDir}`);
await fs.chmod(path.join(androidDir, 'gradlew'), 0o755);

const spawnPromise = spawn(
'bash',
['-c', `./gradlew ${gradleCommand} --profile ${verboseFlag}`],
{
cwd: androidDir,
logger,
lineTransformer: (line?: string) => {
if (!line || /^\.+$/.exec(line)) {
return null;
} else {
return line;
}
},
env: {
...env,
...extraEnv,
LC_ALL: 'C.UTF-8',
},
}
);
if (env.EAS_BUILD_RUNNER === 'eas-build' && process.platform === 'linux') {
adjustOOMScore(spawnPromise, logger);
}

await spawnPromise;
}

/**
* OOM Killer sometimes kills worker server while build is exceeding memory limits.
* `oom_score_adj` is a value between -1000 and 1000 and it defaults to 0.
* It defines which process is more likely to get killed (higher value more likely).
*
* This function sets oom_score_adj for Gradle process and all its child processes.
*/
function adjustOOMScore(spawnPromise: SpawnPromise<SpawnResult>, logger: bunyan): void {
setTimeout(
async () => {
try {
assert(spawnPromise.child.pid);
const pids = await getParentAndDescendantProcessPidsAsync(spawnPromise.child.pid);
await Promise.all(
pids.map(async (pid: number) => {
// Value 800 is just a guess here. It's probably higher than most other
// process. I didn't want to set it any higher, because I'm not sure if OOM Killer
// can start killing processes when there is still enough memory left.
const oomScoreOverride = 800;
await fs.writeFile(`/proc/${pid}/oom_score_adj`, `${oomScoreOverride}\n`);
})
);
} catch (err: any) {
logger.debug({ err, stderr: err?.stderr }, 'Failed to override oom_score_adj');
}
},
// Wait 20 seconds to make sure all child processes are started
20000
);
}

async function getChildrenPidsAsync(parentPids: number[]): Promise<number[]> {
try {
const result = await spawn('pgrep', ['-P', parentPids.join(',')], {
stdio: 'pipe',
});
return result.stdout
.toString()
.split('\n')
.map(i => Number(i.trim()))
.filter(i => i);
} catch {
return [];
}
}

async function getParentAndDescendantProcessPidsAsync(ppid: number): Promise<number[]> {
const children = new Set<number>([ppid]);
let shouldCheckAgain = true;
while (shouldCheckAgain) {
const pids = await getChildrenPidsAsync([...children]);
shouldCheckAgain = false;
for (const pid of pids) {
if (!children.has(pid)) {
shouldCheckAgain = true;
children.add(pid);
}
}
}
return [...children];
}
export { runGradleCommand } from '../../../utils/gradle';

export function resolveGradleCommand(job: Android.Job, command?: string): string {
if (command) {
Expand Down
95 changes: 95 additions & 0 deletions packages/build-tools/src/utils/gradle.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
import { bunyan } from '@expo/logger';
import spawn, { SpawnPromise, SpawnResult } from '@expo/turtle-spawn';
import assert from 'assert';
import fs from 'fs-extra';
import path from 'path';

import { getParentAndDescendantProcessPidsAsync } from './processes';

type GradleEnv = Record<string, string | undefined>;

export async function runGradleCommand({
logger,
gradleCommand,
androidDir,
env,
extraEnv,
}: {
logger: bunyan;
gradleCommand: string;
androidDir: string;
env: GradleEnv;
extraEnv?: GradleEnv;
}): Promise<void> {
const verboseFlag = env['EAS_VERBOSE'] === '1' ? '--info' : '';

logger.info(`Running 'gradlew ${gradleCommand} ${verboseFlag}' in ${androidDir}`);
await fs.chmod(path.join(androidDir, 'gradlew'), 0o755);

const spawnPromise = spawn(
'bash',
['-c', getGradleShellCommand({ gradleCommand, verboseFlag })],
{
cwd: androidDir,
logger,
lineTransformer: (line?: string) => {
if (!line || /^\.+$/.exec(line)) {
return null;
} else {
return line;
}
},
env: {
...env,
...extraEnv,
LC_ALL: 'C.UTF-8',
},
}
);
if (env.EAS_BUILD_RUNNER === 'eas-build' && process.platform === 'linux') {
adjustOOMScore(spawnPromise, logger);
}

await spawnPromise;
}

export function getGradleShellCommand({
gradleCommand,
verboseFlag,
}: {
gradleCommand: string;
verboseFlag: string;
}): string {
return `exec ./gradlew ${gradleCommand} --profile ${verboseFlag}`;
}

/**
* OOM Killer sometimes kills worker server while build is exceeding memory limits.
* `oom_score_adj` is a value between -1000 and 1000 and it defaults to 0.
* It defines which process is more likely to get killed (higher value more likely).
*
* This function sets oom_score_adj for Gradle process and all its child processes.
*/
function adjustOOMScore(spawnPromise: SpawnPromise<SpawnResult>, logger: bunyan): void {
setTimeout(
async () => {
try {
assert(spawnPromise.child.pid);
const pids = await getParentAndDescendantProcessPidsAsync(spawnPromise.child.pid);
await Promise.all(
pids.map(async (pid: number) => {
// Value 800 is just a guess here. It's probably higher than most other
// process. I didn't want to set it any higher, because I'm not sure if OOM Killer
// can start killing processes when there is still enough memory left.
const oomScoreOverride = 800;
await fs.writeFile(`/proc/${pid}/oom_score_adj`, `${oomScoreOverride}\n`);
})
);
} catch (err: any) {
logger.debug({ err, stderr: err?.stderr }, 'Failed to override oom_score_adj');
}
},
// Wait 20 seconds to make sure all child processes are started
20000
);
}
Loading