From 528fe78d31ad9e5b1922eaa3ada45f5dcc496bad Mon Sep 17 00:00:00 2001 From: daz Date: Mon, 8 Apr 2024 13:29:58 -0600 Subject: [PATCH] Moved cache-related code into subpackage --- sources/src/{ => caching}/cache-cleaner.ts | 0 sources/src/caching/cache-key.ts | 97 +++++++++++++++++++ sources/src/{ => caching}/cache-reporting.ts | 0 sources/src/{ => caching}/cache-utils.ts | 95 ------------------ sources/src/{ => caching}/caches.ts | 10 +- .../gradle-home-extry-extractor.ts} | 18 ++-- .../gradle-user-home-cache.ts} | 21 ++-- sources/src/job-summary.ts | 4 +- sources/src/provision.ts | 2 +- sources/src/setup-gradle.ts | 9 +- sources/test/jest/cache-cleanup.test.ts | 2 +- sources/test/jest/cache-debug.test.ts | 6 +- sources/test/jest/cache-reporting.test.ts | 2 +- sources/test/jest/cache-utils.test.ts | 2 +- 14 files changed, 131 insertions(+), 137 deletions(-) rename sources/src/{ => caching}/cache-cleaner.ts (100%) create mode 100644 sources/src/caching/cache-key.ts rename sources/src/{ => caching}/cache-reporting.ts (100%) rename sources/src/{ => caching}/cache-utils.ts (54%) rename sources/src/{ => caching}/caches.ts (90%) rename sources/src/{cache-extract-entries.ts => caching/gradle-home-extry-extractor.ts} (98%) rename sources/src/{cache-base.ts => caching/gradle-user-home-cache.ts} (96%) diff --git a/sources/src/cache-cleaner.ts b/sources/src/caching/cache-cleaner.ts similarity index 100% rename from sources/src/cache-cleaner.ts rename to sources/src/caching/cache-cleaner.ts diff --git a/sources/src/caching/cache-key.ts b/sources/src/caching/cache-key.ts new file mode 100644 index 00000000..12d10e88 --- /dev/null +++ b/sources/src/caching/cache-key.ts @@ -0,0 +1,97 @@ +import * as github from '@actions/github' + +import {CacheConfig, getJobMatrix} from '../input-params' +import {hashStrings} from './cache-utils' + +const CACHE_PROTOCOL_VERSION = 'v9-' + +const CACHE_KEY_PREFIX_VAR = 'GRADLE_BUILD_ACTION_CACHE_KEY_PREFIX' +const CACHE_KEY_OS_VAR = 'GRADLE_BUILD_ACTION_CACHE_KEY_ENVIRONMENT' +const CACHE_KEY_JOB_VAR = 'GRADLE_BUILD_ACTION_CACHE_KEY_JOB' +const CACHE_KEY_JOB_INSTANCE_VAR = 'GRADLE_BUILD_ACTION_CACHE_KEY_JOB_INSTANCE' +const CACHE_KEY_JOB_EXECUTION_VAR = 'GRADLE_BUILD_ACTION_CACHE_KEY_JOB_EXECUTION' + +/** + * Represents a key used to restore a cache entry. + * The Github Actions cache will first try for an exact match on the key. + * If that fails, it will try for a prefix match on any of the restoreKeys. + */ +export class CacheKey { + key: string + restoreKeys: string[] + + constructor(key: string, restoreKeys: string[]) { + this.key = key + this.restoreKeys = restoreKeys + } +} + +/** + * Generates a cache key specific to the current job execution. + * The key is constructed from the following inputs (with some user overrides): + * - The cache protocol version + * - The name of the cache + * - The runner operating system + * - The name of the workflow and Job being executed + * - The matrix values for the Job being executed (job context) + * - The SHA of the commit being executed + * + * Caches are restored by trying to match the these key prefixes in order: + * - The full key with SHA + * - A previous key for this Job + matrix + * - Any previous key for this Job (any matrix) + * - Any previous key for this cache on the current OS + */ +export function generateCacheKey(cacheName: string, config: CacheConfig): CacheKey { + const cacheKeyBase = `${getCacheKeyPrefix()}${CACHE_PROTOCOL_VERSION}${cacheName}` + + // At the most general level, share caches for all executions on the same OS + const cacheKeyForEnvironment = `${cacheKeyBase}|${getCacheKeyEnvironment()}` + + // Then prefer caches that run job with the same ID + const cacheKeyForJob = `${cacheKeyForEnvironment}|${getCacheKeyJob()}` + + // Prefer (even more) jobs that run this job in the same workflow with the same context (matrix) + const cacheKeyForJobContext = `${cacheKeyForJob}[${getCacheKeyJobInstance()}]` + + // Exact match on Git SHA + const cacheKey = `${cacheKeyForJobContext}-${getCacheKeyJobExecution()}` + + if (config.isCacheStrictMatch()) { + return new CacheKey(cacheKey, [cacheKeyForJobContext]) + } + + return new CacheKey(cacheKey, [cacheKeyForJobContext, cacheKeyForJob, cacheKeyForEnvironment]) +} + +export function getCacheKeyPrefix(): string { + // Prefix can be used to force change all cache keys (defaults to cache protocol version) + return process.env[CACHE_KEY_PREFIX_VAR] || '' +} + +function getCacheKeyEnvironment(): string { + const runnerOs = process.env['RUNNER_OS'] || '' + return process.env[CACHE_KEY_OS_VAR] || runnerOs +} + +function getCacheKeyJob(): string { + return process.env[CACHE_KEY_JOB_VAR] || github.context.job +} + +function getCacheKeyJobInstance(): string { + const override = process.env[CACHE_KEY_JOB_INSTANCE_VAR] + if (override) { + return override + } + + // By default, we hash the workflow name and the full `matrix` data for the run, to uniquely identify this job invocation + // The only way we can obtain the `matrix` data is via the `workflow-job-context` parameter in action.yml. + const workflowName = github.context.workflow + const workflowJobContext = getJobMatrix() + return hashStrings([workflowName, workflowJobContext]) +} + +function getCacheKeyJobExecution(): string { + // Used to associate a cache key with a particular execution (default is bound to the git commit sha) + return process.env[CACHE_KEY_JOB_EXECUTION_VAR] || github.context.sha +} diff --git a/sources/src/cache-reporting.ts b/sources/src/caching/cache-reporting.ts similarity index 100% rename from sources/src/cache-reporting.ts rename to sources/src/caching/cache-reporting.ts diff --git a/sources/src/cache-utils.ts b/sources/src/caching/cache-utils.ts similarity index 54% rename from sources/src/cache-utils.ts rename to sources/src/caching/cache-utils.ts index 2759b707..35d0564a 100644 --- a/sources/src/cache-utils.ts +++ b/sources/src/caching/cache-utils.ts @@ -1,6 +1,5 @@ import * as core from '@actions/core' import * as cache from '@actions/cache' -import * as github from '@actions/github' import * as exec from '@actions/exec' import * as crypto from 'crypto' @@ -8,15 +7,6 @@ import * as path from 'path' import * as fs from 'fs' import {CacheEntryListener} from './cache-reporting' -import {CacheConfig, getJobMatrix} from './input-params' - -const CACHE_PROTOCOL_VERSION = 'v9-' - -const CACHE_KEY_PREFIX_VAR = 'GRADLE_BUILD_ACTION_CACHE_KEY_PREFIX' -const CACHE_KEY_OS_VAR = 'GRADLE_BUILD_ACTION_CACHE_KEY_ENVIRONMENT' -const CACHE_KEY_JOB_VAR = 'GRADLE_BUILD_ACTION_CACHE_KEY_JOB' -const CACHE_KEY_JOB_INSTANCE_VAR = 'GRADLE_BUILD_ACTION_CACHE_KEY_JOB_INSTANCE' -const CACHE_KEY_JOB_EXECUTION_VAR = 'GRADLE_BUILD_ACTION_CACHE_KEY_JOB_EXECUTION' const SEGMENT_DOWNLOAD_TIMEOUT_VAR = 'SEGMENT_DOWNLOAD_TIMEOUT_MINS' const SEGMENT_DOWNLOAD_TIMEOUT_DEFAULT = 10 * 60 * 1000 // 10 minutes @@ -28,91 +18,6 @@ export function isCacheDebuggingEnabled(): boolean { return process.env['GRADLE_BUILD_ACTION_CACHE_DEBUG_ENABLED'] ? true : false } -/** - * Represents a key used to restore a cache entry. - * The Github Actions cache will first try for an exact match on the key. - * If that fails, it will try for a prefix match on any of the restoreKeys. - */ -export class CacheKey { - key: string - restoreKeys: string[] - - constructor(key: string, restoreKeys: string[]) { - this.key = key - this.restoreKeys = restoreKeys - } -} - -/** - * Generates a cache key specific to the current job execution. - * The key is constructed from the following inputs (with some user overrides): - * - The cache protocol version - * - The name of the cache - * - The runner operating system - * - The name of the workflow and Job being executed - * - The matrix values for the Job being executed (job context) - * - The SHA of the commit being executed - * - * Caches are restored by trying to match the these key prefixes in order: - * - The full key with SHA - * - A previous key for this Job + matrix - * - Any previous key for this Job (any matrix) - * - Any previous key for this cache on the current OS - */ -export function generateCacheKey(cacheName: string, config: CacheConfig): CacheKey { - const cacheKeyBase = `${getCacheKeyPrefix()}${CACHE_PROTOCOL_VERSION}${cacheName}` - - // At the most general level, share caches for all executions on the same OS - const cacheKeyForEnvironment = `${cacheKeyBase}|${getCacheKeyEnvironment()}` - - // Then prefer caches that run job with the same ID - const cacheKeyForJob = `${cacheKeyForEnvironment}|${getCacheKeyJob()}` - - // Prefer (even more) jobs that run this job in the same workflow with the same context (matrix) - const cacheKeyForJobContext = `${cacheKeyForJob}[${getCacheKeyJobInstance()}]` - - // Exact match on Git SHA - const cacheKey = `${cacheKeyForJobContext}-${getCacheKeyJobExecution()}` - - if (config.isCacheStrictMatch()) { - return new CacheKey(cacheKey, [cacheKeyForJobContext]) - } - - return new CacheKey(cacheKey, [cacheKeyForJobContext, cacheKeyForJob, cacheKeyForEnvironment]) -} - -export function getCacheKeyPrefix(): string { - // Prefix can be used to force change all cache keys (defaults to cache protocol version) - return process.env[CACHE_KEY_PREFIX_VAR] || '' -} - -function getCacheKeyEnvironment(): string { - const runnerOs = process.env['RUNNER_OS'] || '' - return process.env[CACHE_KEY_OS_VAR] || runnerOs -} - -function getCacheKeyJob(): string { - return process.env[CACHE_KEY_JOB_VAR] || github.context.job -} - -function getCacheKeyJobInstance(): string { - const override = process.env[CACHE_KEY_JOB_INSTANCE_VAR] - if (override) { - return override - } - - // By default, we hash the workflow name and the full `matrix` data for the run, to uniquely identify this job invocation - // The only way we can obtain the `matrix` data is via the `workflow-job-context` parameter in action.yml. - const workflowName = github.context.workflow - const workflowJobContext = getJobMatrix() - return hashStrings([workflowName, workflowJobContext]) -} - -function getCacheKeyJobExecution(): string { - // Used to associate a cache key with a particular execution (default is bound to the git commit sha) - return process.env[CACHE_KEY_JOB_EXECUTION_VAR] || github.context.sha -} - export function hashFileNames(fileNames: string[]): string { return hashStrings(fileNames.map(x => x.replace(new RegExp(`\\${path.sep}`, 'g'), '/'))) } diff --git a/sources/src/caches.ts b/sources/src/caching/caches.ts similarity index 90% rename from sources/src/caches.ts rename to sources/src/caching/caches.ts index d2b8ebe0..5b7858dc 100644 --- a/sources/src/caches.ts +++ b/sources/src/caching/caches.ts @@ -1,9 +1,9 @@ import * as core from '@actions/core' import {CacheListener} from './cache-reporting' -import {DaemonController} from './daemon-controller' -import {GradleStateCache} from './cache-base' +import {GradleUserHomeCache} from './gradle-user-home-cache' import {CacheCleaner} from './cache-cleaner' -import {CacheConfig} from './input-params' +import {DaemonController} from '../daemon-controller' +import {CacheConfig} from '../input-params' const CACHE_RESTORED_VAR = 'GRADLE_BUILD_ACTION_CACHE_RESTORED' @@ -20,7 +20,7 @@ export async function restore( } core.exportVariable(CACHE_RESTORED_VAR, true) - const gradleStateCache = new GradleStateCache(userHome, gradleUserHome, cacheConfig) + const gradleStateCache = new GradleUserHomeCache(userHome, gradleUserHome, cacheConfig) if (cacheConfig.isCacheDisabled()) { core.info('Cache is disabled: will not restore state from previous builds.') @@ -99,6 +99,6 @@ export async function save( } await core.group('Caching Gradle state', async () => { - return new GradleStateCache(userHome, gradleUserHome, cacheConfig).save(cacheListener) + return new GradleUserHomeCache(userHome, gradleUserHome, cacheConfig).save(cacheListener) }) } diff --git a/sources/src/cache-extract-entries.ts b/sources/src/caching/gradle-home-extry-extractor.ts similarity index 98% rename from sources/src/cache-extract-entries.ts rename to sources/src/caching/gradle-home-extry-extractor.ts index baf03afa..97edab5a 100644 --- a/sources/src/cache-extract-entries.ts +++ b/sources/src/caching/gradle-home-extry-extractor.ts @@ -4,19 +4,13 @@ import * as core from '@actions/core' import * as glob from '@actions/glob' import * as semver from 'semver' -import {META_FILE_DIR} from './cache-base' +import {META_FILE_DIR} from './gradle-user-home-cache' import {CacheEntryListener, CacheListener} from './cache-reporting' -import { - cacheDebug, - getCacheKeyPrefix, - hashFileNames, - isCacheDebuggingEnabled, - restoreCache, - saveCache, - tryDelete -} from './cache-utils' -import {BuildResult, loadBuildResults} from './build-results' -import {CacheConfig} from './input-params' +import {cacheDebug, hashFileNames, isCacheDebuggingEnabled, restoreCache, saveCache, tryDelete} from './cache-utils' + +import {BuildResult, loadBuildResults} from '../build-results' +import {CacheConfig} from '../input-params' +import {getCacheKeyPrefix} from './cache-key' const SKIP_RESTORE_VAR = 'GRADLE_BUILD_ACTION_SKIP_RESTORE' diff --git a/sources/src/cache-base.ts b/sources/src/caching/gradle-user-home-cache.ts similarity index 96% rename from sources/src/cache-base.ts rename to sources/src/caching/gradle-user-home-cache.ts index 8cce3775..d4353471 100644 --- a/sources/src/cache-base.ts +++ b/sources/src/caching/gradle-user-home-cache.ts @@ -4,29 +4,28 @@ import * as glob from '@actions/glob' import path from 'path' import fs from 'fs' -import {CacheConfig} from './input-params' +import {generateCacheKey} from './cache-key' import {CacheListener} from './cache-reporting' -import {saveCache, restoreCache, cacheDebug, isCacheDebuggingEnabled, tryDelete, generateCacheKey} from './cache-utils' -import {GradleHomeEntryExtractor, ConfigurationCacheEntryExtractor} from './cache-extract-entries' +import {saveCache, restoreCache, cacheDebug, isCacheDebuggingEnabled, tryDelete} from './cache-utils' +import {GradleHomeEntryExtractor, ConfigurationCacheEntryExtractor} from './gradle-home-extry-extractor' +import {CacheConfig} from '../input-params' const RESTORED_CACHE_KEY_KEY = 'restored-cache-key' export const META_FILE_DIR = '.setup-gradle' -export class GradleStateCache { - private cacheConfig: CacheConfig - private cacheName: string - private cacheDescription: string +export class GradleUserHomeCache { + private readonly cacheName = 'gradle' + private readonly cacheDescription = 'Gradle User Home' - protected readonly userHome: string - protected readonly gradleUserHome: string + private readonly userHome: string + private readonly gradleUserHome: string + private readonly cacheConfig: CacheConfig constructor(userHome: string, gradleUserHome: string, cacheConfig: CacheConfig) { this.userHome = userHome this.gradleUserHome = gradleUserHome this.cacheConfig = cacheConfig - this.cacheName = 'gradle' - this.cacheDescription = 'Gradle User Home' } init(): void { diff --git a/sources/src/job-summary.ts b/sources/src/job-summary.ts index 2cf95b98..52afc07d 100644 --- a/sources/src/job-summary.ts +++ b/sources/src/job-summary.ts @@ -3,16 +3,14 @@ import * as github from '@actions/github' import {RequestError} from '@octokit/request-error' import {BuildResult} from './build-results' -import {CacheListener, generateCachingReport} from './cache-reporting' import {SummaryConfig, getGithubToken} from './input-params' export async function generateJobSummary( buildResults: BuildResult[], - cacheListener: CacheListener, + cachingReport: string, config: SummaryConfig ): Promise { const summaryTable = renderSummaryTable(buildResults) - const cachingReport = generateCachingReport(cacheListener) const hasFailure = buildResults.some(result => result.buildFailed) if (config.shouldGenerateJobSummary(hasFailure)) { diff --git a/sources/src/provision.ts b/sources/src/provision.ts index 76369cee..0fd0c474 100644 --- a/sources/src/provision.ts +++ b/sources/src/provision.ts @@ -8,7 +8,7 @@ import * as toolCache from '@actions/tool-cache' import * as gradlew from './gradlew' import * as params from './input-params' -import {handleCacheFailure} from './cache-utils' +import {handleCacheFailure} from './caching/cache-utils' import {CacheConfig} from './input-params' const gradleVersionsBaseUrl = 'https://services.gradle.org/versions' diff --git a/sources/src/setup-gradle.ts b/sources/src/setup-gradle.ts index a6fea8f7..ce9a9890 100644 --- a/sources/src/setup-gradle.ts +++ b/sources/src/setup-gradle.ts @@ -2,13 +2,13 @@ import * as core from '@actions/core' import * as exec from '@actions/exec' import * as path from 'path' import * as os from 'os' -import * as caches from './caches' +import * as caches from './caching/caches' import * as layout from './repository-layout' import * as jobSummary from './job-summary' import * as buildScan from './build-scan' import {loadBuildResults} from './build-results' -import {CacheListener} from './cache-reporting' +import {CacheListener, generateCachingReport} from './caching/cache-reporting' import {DaemonController} from './daemon-controller' import {BuildScanConfig, CacheConfig, SummaryConfig} from './input-params' @@ -57,11 +57,12 @@ export async function complete(cacheConfig: CacheConfig, summaryConfig: SummaryC const userHome = core.getState(USER_HOME) const gradleUserHome = core.getState(GRADLE_USER_HOME) const cacheListener: CacheListener = CacheListener.rehydrate(core.getState(CACHE_LISTENER)) - const daemonController = new DaemonController(buildResults) + const daemonController = new DaemonController(buildResults) await caches.save(userHome, gradleUserHome, cacheListener, daemonController, cacheConfig) - await jobSummary.generateJobSummary(buildResults, cacheListener, summaryConfig) + const cachingReport = generateCachingReport(cacheListener) + await jobSummary.generateJobSummary(buildResults, cachingReport, summaryConfig) core.info('Completed post-action step') diff --git a/sources/test/jest/cache-cleanup.test.ts b/sources/test/jest/cache-cleanup.test.ts index 9ef4b9f2..0cc94532 100644 --- a/sources/test/jest/cache-cleanup.test.ts +++ b/sources/test/jest/cache-cleanup.test.ts @@ -1,7 +1,7 @@ import * as exec from '@actions/exec' import fs from 'fs' import path from 'path' -import {CacheCleaner} from '../../src/cache-cleaner' +import {CacheCleaner} from '../../src/caching/cache-cleaner' jest.setTimeout(120000) diff --git a/sources/test/jest/cache-debug.test.ts b/sources/test/jest/cache-debug.test.ts index f0466a53..d0c989de 100644 --- a/sources/test/jest/cache-debug.test.ts +++ b/sources/test/jest/cache-debug.test.ts @@ -1,6 +1,6 @@ import * as path from 'path' import * as fs from 'fs' -import {GradleStateCache} from "../../src/cache-base" +import {GradleUserHomeCache} from "../../src/caching/gradle-user-home-cache" import {CacheConfig} from "../../src/input-params" const testTmp = 'test/jest/tmp' @@ -12,7 +12,7 @@ describe("--info and --stacktrace", () => { const emptyGradleHome = `${testTmp}/empty-gradle-home` fs.mkdirSync(emptyGradleHome, {recursive: true}) - const stateCache = new GradleStateCache("ignored", emptyGradleHome, new CacheConfig()) + const stateCache = new GradleUserHomeCache("ignored", emptyGradleHome, new CacheConfig()) stateCache.configureInfoLogLevel() expect(fs.readFileSync(path.resolve(emptyGradleHome, "gradle.properties"), 'utf-8')) @@ -25,7 +25,7 @@ describe("--info and --stacktrace", () => { fs.mkdirSync(existingGradleHome, {recursive: true}) fs.writeFileSync(path.resolve(existingGradleHome, "gradle.properties"), "org.gradle.logging.level=debug\n") - const stateCache = new GradleStateCache("ignored", existingGradleHome, new CacheConfig()) + const stateCache = new GradleUserHomeCache("ignored", existingGradleHome, new CacheConfig()) stateCache.configureInfoLogLevel() expect(fs.readFileSync(path.resolve(existingGradleHome, "gradle.properties"), 'utf-8')) diff --git a/sources/test/jest/cache-reporting.test.ts b/sources/test/jest/cache-reporting.test.ts index 30f0c207..d728e6c4 100644 --- a/sources/test/jest/cache-reporting.test.ts +++ b/sources/test/jest/cache-reporting.test.ts @@ -1,4 +1,4 @@ -import {CacheEntryListener, CacheListener} from '../../src/cache-reporting' +import {CacheEntryListener, CacheListener} from '../../src/caching/cache-reporting' describe('caching report', () => { describe('reports not fully restored', () => { diff --git a/sources/test/jest/cache-utils.test.ts b/sources/test/jest/cache-utils.test.ts index 6ef38ff5..ac6fb108 100644 --- a/sources/test/jest/cache-utils.test.ts +++ b/sources/test/jest/cache-utils.test.ts @@ -1,4 +1,4 @@ -import * as cacheUtils from '../../src/cache-utils' +import * as cacheUtils from '../../src/caching/cache-utils' describe('cacheUtils-utils', () => { describe('can hash', () => {