From 169bec5d8b1a645e16b675d6b7a5d4cc444eac02 Mon Sep 17 00:00:00 2001 From: daz Date: Fri, 28 Jun 2024 12:39:09 -0600 Subject: [PATCH 1/4] Provision latest Gradle for cache-cleanup To cleanup Gradle User Home, a Gradle build must be executed. Newer Gradle versions are able to cleanup the home directories of older versions, but not vice-versa. With this change, the latest version of Gradle is automatically provisioned in order to run Gradle User Home cleanup. This ensures a consistent version of Gradle is used for cleanup, and fixes #33 where Gradle is not pre-installed on a custom runner. --- sources/src/caching/cache-cleaner.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/sources/src/caching/cache-cleaner.ts b/sources/src/caching/cache-cleaner.ts index 26c64a7d..8081113c 100644 --- a/sources/src/caching/cache-cleaner.ts +++ b/sources/src/caching/cache-cleaner.ts @@ -1,8 +1,8 @@ import * as core from '@actions/core' -import * as exec from '@actions/exec' import * as glob from '@actions/glob' import fs from 'fs' import path from 'path' +import {provisionAndMaybeExecute} from '../execution/gradle' export class CacheCleaner { private readonly gradleUserHome: string @@ -42,10 +42,16 @@ export class CacheCleaner { ) fs.writeFileSync(path.resolve(cleanupProjectDir, 'build.gradle'), 'task("noop") {}') - const gradleCommand = `gradle -g ${this.gradleUserHome} --no-daemon --build-cache --no-scan --quiet -DGITHUB_DEPENDENCY_GRAPH_ENABLED=false noop` - await exec.exec(gradleCommand, [], { - cwd: cleanupProjectDir - }) + await provisionAndMaybeExecute('current', cleanupProjectDir, [ + '-g', + this.gradleUserHome, + '--quiet', + '--no-daemon', + '--no-scan', + '--build-cache', + '-DGITHUB_DEPENDENCY_GRAPH_ENABLED=false', + 'noop' + ]) } private async ageAllFiles(fileName = '*'): Promise { From 95ef72241ee592cf868f27666dd5dda1e7fefc14 Mon Sep 17 00:00:00 2001 From: daz Date: Thu, 21 Mar 2024 19:03:05 -0600 Subject: [PATCH 2/4] Use Gradle 8.8 features for cleanup Gradle 8.8 introduces new features that allow us to avoid using timestamp manipulation to force the cleanup of the Gradle User Home directory. This solution is simpler and more robust, but relies on Gradle 8.8+ always being used for the cache cleanup operation. Fixes #24 --- sources/src/caching/cache-cleaner.ts | 67 ++++++++++++------------- sources/test/jest/cache-cleanup.test.ts | 33 +++++++++--- 2 files changed, 58 insertions(+), 42 deletions(-) diff --git a/sources/src/caching/cache-cleaner.ts b/sources/src/caching/cache-cleaner.ts index 8081113c..20f3e0bc 100644 --- a/sources/src/caching/cache-cleaner.ts +++ b/sources/src/caching/cache-cleaner.ts @@ -1,5 +1,4 @@ import * as core from '@actions/core' -import * as glob from '@actions/glob' import fs from 'fs' import path from 'path' import {provisionAndMaybeExecute} from '../execution/gradle' @@ -13,25 +12,20 @@ export class CacheCleaner { this.tmpDir = tmpDir } - async prepare(): Promise { - // Reset the file-access journal so that files appear not to have been used recently - fs.rmSync(path.resolve(this.gradleUserHome, 'caches/journal-1'), {recursive: true, force: true}) - fs.mkdirSync(path.resolve(this.gradleUserHome, 'caches/journal-1'), {recursive: true}) - fs.writeFileSync( - path.resolve(this.gradleUserHome, 'caches/journal-1/file-access.properties'), - 'inceptionTimestamp=0' - ) - - // Set the modification time of all files to the past: this timestamp is used when there is no matching entry in the journal - await this.ageAllFiles() - - // Touch all 'gc' files so that cache cleanup won't run immediately. - await this.touchAllFiles('gc.properties') + async prepare(): Promise { + // Save the current timestamp + const timestamp = Date.now().toString() + core.saveState('clean-timestamp', timestamp) + return timestamp } async forceCleanup(): Promise { - // Age all 'gc' files so that cache cleanup will run immediately. - await this.ageAllFiles('gc.properties') + const cleanTimestamp = core.getState('clean-timestamp') + await this.forceCleanupFilesOlderThan(cleanTimestamp) + } + + async forceCleanupFilesOlderThan(cleanTimestamp: string): Promise { + core.info(`Cleaning up caches before ${cleanTimestamp}`) // Run a dummy Gradle build to trigger cache cleanup const cleanupProjectDir = path.resolve(this.tmpDir, 'dummy-cleanup-project') @@ -40,11 +34,31 @@ export class CacheCleaner { path.resolve(cleanupProjectDir, 'settings.gradle'), 'rootProject.name = "dummy-cleanup-project"' ) + fs.writeFileSync( + path.resolve(cleanupProjectDir, 'init.gradle'), + ` + beforeSettings { settings -> + def cleanupTime = ${cleanTimestamp} + + settings.caches { + cleanup = Cleanup.ALWAYS + + releasedWrappers.removeUnusedEntriesOlderThan.set(cleanupTime) + snapshotWrappers.removeUnusedEntriesOlderThan.set(cleanupTime) + downloadedResources.removeUnusedEntriesOlderThan.set(cleanupTime) + createdResources.removeUnusedEntriesOlderThan.set(cleanupTime) + buildCache.removeUnusedEntriesOlderThan.set(cleanupTime) + } + } + ` + ) fs.writeFileSync(path.resolve(cleanupProjectDir, 'build.gradle'), 'task("noop") {}') await provisionAndMaybeExecute('current', cleanupProjectDir, [ '-g', this.gradleUserHome, + '-I', + 'init.gradle', '--quiet', '--no-daemon', '--no-scan', @@ -53,23 +67,4 @@ export class CacheCleaner { 'noop' ]) } - - private async ageAllFiles(fileName = '*'): Promise { - core.debug(`Aging all files in Gradle User Home with name ${fileName}`) - await this.setUtimes(`${this.gradleUserHome}/**/${fileName}`, new Date(0)) - } - - private async touchAllFiles(fileName = '*'): Promise { - core.debug(`Touching all files in Gradle User Home with name ${fileName}`) - await this.setUtimes(`${this.gradleUserHome}/**/${fileName}`, new Date()) - } - - private async setUtimes(pattern: string, timestamp: Date): Promise { - const globber = await glob.create(pattern, { - implicitDescendants: false - }) - for await (const file of globber.globGenerator()) { - fs.utimesSync(file, timestamp, timestamp) - } - } } diff --git a/sources/test/jest/cache-cleanup.test.ts b/sources/test/jest/cache-cleanup.test.ts index f97dbd87..2eed9a12 100644 --- a/sources/test/jest/cache-cleanup.test.ts +++ b/sources/test/jest/cache-cleanup.test.ts @@ -1,5 +1,6 @@ import * as exec from '@actions/exec' import * as core from '@actions/core' +import * as glob from '@actions/glob' import fs from 'fs' import path from 'path' import {CacheCleaner} from '../../src/caching/cache-cleaner' @@ -14,7 +15,7 @@ test('will cleanup unused dependency jars and build-cache entries', async () => await runGradleBuild(projectRoot, 'build', '3.1') - await cacheCleaner.prepare() + const timestamp = await cacheCleaner.prepare() await runGradleBuild(projectRoot, 'build', '3.1.1') @@ -26,7 +27,7 @@ test('will cleanup unused dependency jars and build-cache entries', async () => expect(fs.existsSync(commonsMath311)).toBe(true) expect(fs.readdirSync(buildCacheDir).length).toBe(4) // gc.properties, build-cache-1.lock, and 2 task entries - await cacheCleaner.forceCleanup() + await cacheCleaner.forceCleanupFilesOlderThan(timestamp) expect(fs.existsSync(commonsMath31)).toBe(false) expect(fs.existsSync(commonsMath311)).toBe(true) @@ -42,25 +43,39 @@ test('will cleanup unused gradle versions', async () => { // Initialize HOME with 2 different Gradle versions await runGradleWrapperBuild(projectRoot, 'build') await runGradleBuild(projectRoot, 'build') - - await cacheCleaner.prepare() + + const timestamp = await cacheCleaner.prepare() // Run with only one of these versions await runGradleBuild(projectRoot, 'build') const gradle802 = path.resolve(gradleUserHome, "caches/8.0.2") + const transforms3 = path.resolve(gradleUserHome, "caches/transforms-3") + const metadata100 = path.resolve(gradleUserHome, "caches/modules-2/metadata-2.100") const wrapper802 = path.resolve(gradleUserHome, "wrapper/dists/gradle-8.0.2-bin") const gradleCurrent = path.resolve(gradleUserHome, "caches/8.8") + const metadataCurrent = path.resolve(gradleUserHome, "caches/modules-2/metadata-2.106") expect(fs.existsSync(gradle802)).toBe(true) + expect(fs.existsSync(transforms3)).toBe(true) + expect(fs.existsSync(metadata100)).toBe(true) expect(fs.existsSync(wrapper802)).toBe(true) - expect(fs.existsSync(gradleCurrent)).toBe(true) - await cacheCleaner.forceCleanup() + expect(fs.existsSync(gradleCurrent)).toBe(true) + expect(fs.existsSync(metadataCurrent)).toBe(true) + + // The wrapper won't be removed if it was recently downloaded. Age it. + setUtimes(wrapper802, new Date(Date.now() - 48 * 60 * 60 * 1000)) + + await cacheCleaner.forceCleanupFilesOlderThan(timestamp) expect(fs.existsSync(gradle802)).toBe(false) + expect(fs.existsSync(transforms3)).toBe(false) + expect(fs.existsSync(metadata100)).toBe(false) expect(fs.existsSync(wrapper802)).toBe(false) + expect(fs.existsSync(gradleCurrent)).toBe(true) + expect(fs.existsSync(metadataCurrent)).toBe(true) }) async function runGradleBuild(projectRoot: string, args: string, version: string = '3.1'): Promise { @@ -86,3 +101,9 @@ function prepareTestProject(): string { return projectRoot } +async function setUtimes(pattern: string, timestamp: Date): Promise { + const globber = await glob.create(pattern) + for await (const file of globber.globGenerator()) { + fs.utimesSync(file, timestamp, timestamp) + } +} From 4022faad7e8d0f0fd3701bac04f42372af22ba59 Mon Sep 17 00:00:00 2001 From: daz Date: Fri, 28 Jun 2024 13:21:54 -0600 Subject: [PATCH 3/4] Fix integ-test-cache-cleanup.yml for running on act --- .github/workflows/integ-test-cache-cleanup.yml | 12 +++++++++--- CONTRIBUTING.md | 1 - 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integ-test-cache-cleanup.yml b/.github/workflows/integ-test-cache-cleanup.yml index c6d44054..060547d1 100644 --- a/.github/workflows/integ-test-cache-cleanup.yml +++ b/.github/workflows/integ-test-cache-cleanup.yml @@ -35,7 +35,7 @@ jobs: cache-read-only: false # For testing, allow writing cache entries on non-default branches - name: Build with 3.1 working-directory: sources/test/jest/resources/cache-cleanup - run: gradle --no-daemon --build-cache -Dcommons_math3_version="3.1" build + run: ./gradlew --no-daemon --build-cache -Dcommons_math3_version="3.1" build # Second build will use the cache from the first build, but cleanup should remove unused artifacts assemble-build: @@ -58,7 +58,7 @@ jobs: gradle-home-cache-cleanup: true - name: Build with 3.1.1 working-directory: sources/test/jest/resources/cache-cleanup - run: gradle --no-daemon --build-cache -Dcommons_math3_version="3.1.1" build + run: ./gradlew --no-daemon --build-cache -Dcommons_math3_version="3.1.1" build check-clean-cache: needs: assemble-build @@ -78,7 +78,9 @@ jobs: with: cache-read-only: true - name: Report Gradle User Home - run: du -hc ~/.gradle/caches/modules-2 + run: | + du -hc ~/.gradle/caches/modules-2 + du -hc ~/.gradle/wrapper/dists - name: Verify cleaned cache shell: bash run: | @@ -90,3 +92,7 @@ jobs: echo "::error ::Should NOT find commons-math3 3.1 in cache" exit 1 fi + if [ ! -e ~/.gradle/wrapper/dists/gradle-8.0.2-bin ]; then + echo "::error ::Should find gradle-8.0.2 in wrapper/dists" + exit 1 + fi diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6c3313d1..2fed3d08 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,6 @@ Example running a single job: `./build act -W .github/workflows/integ-test-caching-config.yml -j cache-disabled-pre-existing-gradle-home` Known issues: -- `integ-test-cache-cleanup.yml` fails because `gradle` is not installed on the runner. Should be fixed by #33. - `integ-test-detect-java-toolchains.yml` fails when running on a `linux/amd64` container, since the expected pre-installed JDKs are not present. Should be fixed by #89. - `act` is not yet compatible with `actions/upload-artifact@v4` (or related toolkit functions) - See https://github.com/nektos/act/pull/2224 From 621f3b3f79204b94d98c9da8c00526d3c8f82d84 Mon Sep 17 00:00:00 2001 From: daz Date: Fri, 28 Jun 2024 13:25:56 -0600 Subject: [PATCH 4/4] Run cache-cleanup build with --info Resolves #169 --- sources/src/caching/cache-cleaner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sources/src/caching/cache-cleaner.ts b/sources/src/caching/cache-cleaner.ts index 20f3e0bc..12e3be59 100644 --- a/sources/src/caching/cache-cleaner.ts +++ b/sources/src/caching/cache-cleaner.ts @@ -59,7 +59,7 @@ export class CacheCleaner { this.gradleUserHome, '-I', 'init.gradle', - '--quiet', + '--info', '--no-daemon', '--no-scan', '--build-cache',