From a7f7b7666d82bd3f282dafe1d241b1e19693c5d0 Mon Sep 17 00:00:00 2001 From: Adam Wathan Date: Fri, 12 May 2023 10:13:29 -0400 Subject: [PATCH] [Oxide] Automatic content detection (#11173) * resolve all _existing_ content paths * pin `@napi-rs/cli` * WIP: Log all resolved content files/globs * only filter out raw changed content in non-auto mode * skip parseCandidateFiles cache in `auto` mode * improve algorithm of detecting content paths 1. Files in the root should be listed statically instead of using globs. 2. Files and folders in special known direct child folders should be listed statically instead of using globs (e.g.: `public`). This is because these special folders are often used to store generated AND source files at the same time. Using globs could trigger infinite loops because we are watching and acting upon dist files. 3. All file extensions found in the project, should be used in the globs in addition to a known set of extensions. 4. Direct folders seen from the root, can use the glob syntax `/src/**/*.{...known-extensions}` * inline wanted-extensions Not 100% convinced yet, but seems cleaner so far. * ensure writing an file also makes the parent folder(s) * add integration tests for the auto content feature * add pnpm and bun lock files * Revert "inline wanted-extensions" This reverts commit 879c1248524e84216125f4a24e0160b40736333a. * sort binary-extensions and add lockb * sort + add `lock` to ignored extensions * drop `yarn.lock`, because lock extensions are already covered * group template extensions This will make it a bit easier to organize in the future. * drop empty lines and commented lines from template-extensions * skip the config path when resolving template files The config file will automatically trigger a rebuild when this file is changed. However, this should not be part of the template files because that could cause additional css that's not being used. * make `auto content` the default in the oxide engine - In the oxide engine, the default `content: []` will be dropped from the default configuration (config.simple.js, config.full.js). - If you have `content: []` or `content: { files: [] }` then the auto content feature won't be active. However if those arrays are empty a warning will still be shown. Adding files/globs or dropping the `content` section completely will enable auto content. * only test the auto content integration test in the oxide engine * set `content.files` to `auto` instead of using `auto: boolean` This way we don't run into the issue where the `config.content.files` is set and the `config.content.auto` is set to true. * drop log * ensure we validate the config in the CLI * show experimental warning for automatic content detection * use cached version of the getCandidateFiles instead of bypassing it * use `is_empty()` shorthand Thanks, Clippy! * add test to ensure nested ignored folders are not scanned * add `tempfile` for tests * add auto content tests in Rust * refactor auto content detection This will also make sure that if we have (deeply) nested ignored folders, then we won't use deeply nested globs (**/*.{js,html}) for the parent(s) of the nested ignored folders but instead use a shallow glob for each directory (*/*.{js,html}). Then each sibling directory of the parent can use deeply nested globs again except for the direct parent. * use consistent comments * ensure ignored static listed files are not present * improve performance by ~30x On a big test project this goes from ~6s to ~200ms * improve performance by ~5x We started with a ~6s duration Then in the previous commit, we improved it by ~30x and it went down to ~200ms Now with this change, it takes about ~40ms. That's another ~5x improvement. Or in total a ~150x improvement. * ensure nested folders in `public/` are also explicitly listed * add shortcut for normalizing files This is only called once so won't do anything to the main performance of Tailwind CSS. But always nice to make small performance improvements! * run Rust tests in CI * fix lint warnings * update changelog * Update CHANGELOG.md --------- Co-authored-by: Robin Malfait --- .github/workflows/ci.yml | 10 +- CHANGELOG.md | 1 + integrations/io.js | 11 + .../fixtures/example-app/src/index.css | 3 + .../fixtures/example-app/src/index.html | 1 + .../tailwindcss-cli/tests/integration.test.js | 433 ++++++++++++++++++ oxide/Cargo.lock | 269 ++++++++++- oxide/crates/core/Cargo.toml | 3 + .../core/src/fixtures/binary-extensions.txt | 259 +++++++++++ .../core/src/fixtures/ignored-extensions.txt | 6 + .../core/src/fixtures/ignored-files.txt | 3 + .../core/src/fixtures/template-extensions.txt | 57 +++ oxide/crates/core/src/lib.rs | 281 +++++++++++- oxide/crates/core/src/parser.rs | 4 +- oxide/crates/core/tests/auto_content.rs | 248 ++++++++++ oxide/crates/node/package.json | 2 +- oxide/crates/node/src/lib.rs | 11 + package-lock.json | 6 +- src/cli/build/plugin.js | 9 +- src/cli/init/index.js | 5 + src/lib/content.js | 35 +- src/oxide/cli/build/plugin.ts | 9 +- src/oxide/cli/init/index.ts | 5 + src/util/getAllConfigs.js | 9 +- src/util/normalizeConfig.js | 68 ++- src/util/validateConfig.js | 9 +- tests/normalize-config.test.js | 4 +- tests/warnings.test.js | 6 +- 28 files changed, 1714 insertions(+), 53 deletions(-) create mode 100644 integrations/tailwindcss-cli/fixtures/example-app/src/index.css create mode 100644 integrations/tailwindcss-cli/fixtures/example-app/src/index.html create mode 100644 oxide/crates/core/src/fixtures/binary-extensions.txt create mode 100644 oxide/crates/core/src/fixtures/ignored-extensions.txt create mode 100644 oxide/crates/core/src/fixtures/ignored-files.txt create mode 100644 oxide/crates/core/src/fixtures/template-extensions.txt create mode 100644 oxide/crates/core/tests/auto_content.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e2433d3b7ae..b7e4a467466e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,11 +65,19 @@ jobs: - name: Build Tailwind CSS run: npx turbo run build --filter=// - - name: Test + - name: Test (Node.js) run: | npx turbo run test --filter=// || \ npx turbo run test --filter=// || \ npx turbo run test --filter=// || exit 1 + - name: Test (Rust) + run: | + cd ./oxide + cargo test || \ + cargo test || \ + cargo test || exit 1 + cd - + - name: Lint run: npx turbo run style --filter=// diff --git a/CHANGELOG.md b/CHANGELOG.md index d528fade1d7b..39806a3dc728 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [Oxide] Use `lightningcss` for nesting and vendor prefixes in PostCSS plugin ([#10399](https://github.com/tailwindlabs/tailwindcss/pull/10399)) - Support `@import "tailwindcss"` using top-level `index.css` file ([#11205](https://github.com/tailwindlabs/tailwindcss/pull/11205)) +- [Oxide] Automatically detect content paths when no `content` configuration is provided ([#11173](https://github.com/tailwindlabs/tailwindcss/pull/11173)) ### Changed diff --git a/integrations/io.js b/integrations/io.js index c555ac8ba02c..d30570d3afce 100644 --- a/integrations/io.js +++ b/integrations/io.js @@ -122,6 +122,17 @@ module.exports = function ({ }, async writeInputFile(file, contents) { let filePath = path.resolve(absoluteInputFolder, file) + + // Ensure the parent folder of the file exists + if ( + !(await fs + .stat(filePath) + .then(() => true) + .catch(() => false)) + ) { + await fs.mkdir(path.dirname(filePath), { recursive: true }) + } + if (!fileCache[filePath]) { try { fileCache[filePath] = await fs.readFile(filePath, 'utf8') diff --git a/integrations/tailwindcss-cli/fixtures/example-app/src/index.css b/integrations/tailwindcss-cli/fixtures/example-app/src/index.css new file mode 100644 index 000000000000..b5c61c956711 --- /dev/null +++ b/integrations/tailwindcss-cli/fixtures/example-app/src/index.css @@ -0,0 +1,3 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; diff --git a/integrations/tailwindcss-cli/fixtures/example-app/src/index.html b/integrations/tailwindcss-cli/fixtures/example-app/src/index.html new file mode 100644 index 000000000000..b956240c2a38 --- /dev/null +++ b/integrations/tailwindcss-cli/fixtures/example-app/src/index.html @@ -0,0 +1 @@ +
\ No newline at end of file diff --git a/integrations/tailwindcss-cli/tests/integration.test.js b/integrations/tailwindcss-cli/tests/integration.test.js index 75c9779af8f8..de806dd27524 100644 --- a/integrations/tailwindcss-cli/tests/integration.test.js +++ b/integrations/tailwindcss-cli/tests/integration.test.js @@ -1,4 +1,5 @@ let fs = require('fs') +let path = require('path') let $ = require('../../execute') let { css, html, javascript } = require('../../syntax') let { env } = require('../../../lib/lib/sharedState') @@ -1000,4 +1001,436 @@ describe('watcher', () => { return runningProcess.stop() }) + + if (env.ENGINE === 'oxide') { + describe('auto content', () => { + let { readOutputFile, writeInputFile } = require('../../io')({ + output: 'fixtures/example-app/dist', + input: 'fixtures/example-app/src', + }) + let options = { + cwd: path.resolve(__dirname, '..', 'fixtures', 'example-app'), + } + + it('should detect classes in existing files', async () => { + await writeInputFile( + '../tailwind.config.js', + javascript` + module.exports = { + corePlugins: { + preflight: false, + }, + } + ` + ) + + let runningProcess = $( + 'node ../../../../lib/cli.js -i ./src/index.css -o ./dist/main.css -w', + options + ) + + await writeInputFile('index.html', html`
`) + await runningProcess.onStderr(ready) + + expect(await readOutputFile('main.css')).toIncludeCss( + css` + .font-bold { + font-weight: 700; + } + ` + ) + + return runningProcess.stop() + }) + + it('should detect changes in existing files', async () => { + await writeInputFile( + '../tailwind.config.js', + javascript` + module.exports = { + corePlugins: { + preflight: false, + }, + } + ` + ) + + await writeInputFile('index.html', html`
`) + + let runningProcess = $( + 'node ../../../../lib/cli.js -i ./src/index.css -o ./dist/main.css -w', + options + ) + await runningProcess.onStderr(ready) + + expect(await readOutputFile('main.css')).toIncludeCss( + css` + .font-bold { + font-weight: 700; + } + ` + ) + + expect(await readOutputFile('main.css')).not.toIncludeCss( + css` + .underline { + -webkit-text-decoration-line: underline; + text-decoration-line: underline; + } + ` + ) + + // Make a change + + await writeInputFile('index.html', html`
`) + await runningProcess.onStderr(ready) + + expect(await readOutputFile('main.css')).toIncludeCss( + css` + .underline { + -webkit-text-decoration-line: underline; + text-decoration-line: underline; + } + ` + ) + + return runningProcess.stop() + }) + + it('should detect changes in new files in existing folders with a known extension', async () => { + await writeInputFile( + '../tailwind.config.js', + javascript` + module.exports = { + corePlugins: { + preflight: false, + }, + } + ` + ) + + await writeInputFile('index.html', html`
`) + + let runningProcess = $( + 'node ../../../../lib/cli.js -i ./src/index.css -o ./dist/main.css -w', + options + ) + await runningProcess.onStderr(ready) + + expect(await readOutputFile('main.css')).toIncludeCss( + css` + .font-bold { + font-weight: 700; + } + ` + ) + + expect(await readOutputFile('main.css')).not.toIncludeCss( + css` + .underline { + -webkit-text-decoration-line: underline; + text-decoration-line: underline; + } + ` + ) + + // Make a change to a new file in an existing folder with a known extension. + + await writeInputFile('other.html', html`
`) + await runningProcess.onStderr(ready) + + expect(await readOutputFile('main.css')).toIncludeCss( + css` + .underline { + -webkit-text-decoration-line: underline; + text-decoration-line: underline; + } + ` + ) + + return runningProcess.stop() + }) + + it('should not scan ignored files', async () => { + await writeInputFile( + '../tailwind.config.js', + javascript` + module.exports = { + corePlugins: { + preflight: false, + }, + } + ` + ) + await writeInputFile('../.gitignore', 'generated-folder/') + await writeInputFile('../generated-folder/bad.html', html`
`) + await writeInputFile('index.html', html`
`) + + let runningProcess = $( + 'node ../../../../lib/cli.js -i ./src/index.css -o ./dist/main.css -w', + options + ) + await runningProcess.onStderr(ready) + + expect(await readOutputFile('main.css')).toIncludeCss( + css` + .font-bold { + font-weight: 700; + } + ` + ) + + expect(await readOutputFile('main.css')).not.toIncludeCss( + css` + .italic { + font-style: italic; + } + ` + ) + + return runningProcess.stop() + }) + + it('should not scan for known binary files', async () => { + await writeInputFile( + '../tailwind.config.js', + javascript` + module.exports = { + corePlugins: { + preflight: false, + }, + } + ` + ) + await writeInputFile('example-1.png', html`
`) + await writeInputFile('example-2.mp4', html`
`) + await writeInputFile('index.html', html`
`) + + let runningProcess = $( + 'node ../../../../lib/cli.js -i ./src/index.css -o ./dist/main.css -w', + options + ) + await runningProcess.onStderr(ready) + + expect(await readOutputFile('main.css')).toIncludeCss( + css` + .font-bold { + font-weight: 700; + } + ` + ) + + expect(await readOutputFile('main.css')).not.toIncludeCss( + css` + .italic { + font-style: italic; + } + + .underline { + -webkit-text-decoration-line: underline; + text-decoration-line: underline; + } + ` + ) + + return runningProcess.stop() + }) + + it('should not scan for explicitly ignored extensions (such as css/scss/less/...)', async () => { + await writeInputFile( + '../tailwind.config.js', + javascript` + module.exports = { + corePlugins: { + preflight: false, + }, + } + ` + ) + await writeInputFile('example.css', html`
`) + await writeInputFile('example.less', html`
`) + await writeInputFile('index.html', html`
`) + + let runningProcess = $( + 'node ../../../../lib/cli.js -i ./src/index.css -o ./dist/main.css -w', + options + ) + await runningProcess.onStderr(ready) + + expect(await readOutputFile('main.css')).toIncludeCss( + css` + .font-bold { + font-weight: 700; + } + ` + ) + + expect(await readOutputFile('main.css')).not.toIncludeCss( + css` + .italic { + font-style: italic; + } + + .underline { + -webkit-text-decoration-line: underline; + text-decoration-line: underline; + } + ` + ) + + return runningProcess.stop() + }) + + it('should not scan for explicitly ignored files (such as package-lock.json)', async () => { + await writeInputFile( + '../tailwind.config.js', + javascript` + module.exports = { + corePlugins: { + preflight: false, + }, + } + ` + ) + await writeInputFile('package-lock.json', html`
`) + await writeInputFile('yarn.lock', html`
`) + await writeInputFile('pnpm-lock.yaml', html`
`) + await writeInputFile('index.html', html`
`) + + let runningProcess = $( + 'node ../../../../lib/cli.js -i ./src/index.css -o ./dist/main.css -w', + options + ) + await runningProcess.onStderr(ready) + + expect(await readOutputFile('main.css')).toIncludeCss( + css` + .font-bold { + font-weight: 700; + } + ` + ) + + expect(await readOutputFile('main.css')).not.toIncludeCss( + css` + .italic { + font-style: italic; + } + ` + ) + + return runningProcess.stop() + }) + + it('should not include the tailwind.config.js file as a template file', async () => { + await writeInputFile( + '../tailwind.config.js', + javascript` + // Example class that should not be included: flex italic + module.exports = { + corePlugins: { + preflight: false, + }, + } + ` + ) + await writeInputFile('index.html', html`
`) + + let runningProcess = $( + 'node ../../../../lib/cli.js -i ./src/index.css -o ./dist/main.css -w', + options + ) + await runningProcess.onStderr(ready) + + expect(await readOutputFile('main.css')).toIncludeCss( + css` + .font-bold { + font-weight: 700; + } + ` + ) + + expect(await readOutputFile('main.css')).not.toIncludeCss( + css` + .flex { + display: flex; + } + ` + ) + expect(await readOutputFile('main.css')).not.toIncludeCss( + css` + .italic { + font-style: italic; + } + ` + ) + + return runningProcess.stop() + }) + + it('should optimize the globs and ensure that nested ignored folders are not scanned', async () => { + await writeInputFile( + '../tailwind.config.js', + javascript` + module.exports = { + corePlugins: { + preflight: false, + }, + } + ` + ) + + await writeInputFile('../.gitignore', 'node_modules') + await writeInputFile('../node_modules/a.html', html`
`) + await writeInputFile('index.html', html`
`) + await writeInputFile('nested/index.html', html`
`) + await writeInputFile( + 'nested/node_modules/index.html', + html`
` + ) + + let runningProcess = $( + 'node ../../../../lib/cli.js -i ./src/index.css -o ./dist/main.css -w', + options + ) + await runningProcess.onStderr(ready) + + // Root node_modules + expect(await readOutputFile('main.css')).not.toIncludeCss( + css` + .text-red-100 { + color: #fee2e2; + } + ` + ) + + expect(await readOutputFile('main.css')).toIncludeCss( + css` + .text-red-200 { + color: #fecaca; + } + ` + ) + + expect(await readOutputFile('main.css')).toIncludeCss( + css` + .text-red-300 { + color: #fca5a5; + } + ` + ) + + // Nested node_modules + expect(await readOutputFile('main.css')).not.toIncludeCss( + css` + .text-red-400 { + color: #f87171; + } + ` + ) + + return runningProcess.stop() + }) + }) + } }) diff --git a/oxide/Cargo.lock b/oxide/Cargo.lock index 5b959d72682a..948b8b4cc739 100644 --- a/oxide/Cargo.lock +++ b/oxide/Cargo.lock @@ -17,7 +17,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] @@ -48,9 +48,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.0.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca0852af221f458706eb0725c03e4ed6c46af9ac98e6a689d5e634215d594dd" +checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09" dependencies = [ "memchr", "once_cell", @@ -76,6 +76,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cc" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" + [[package]] name = "cfg-if" version = "1.0.0" @@ -280,6 +286,36 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + [[package]] name = "fnv" version = "1.0.7" @@ -297,12 +333,12 @@ dependencies = [ [[package]] name = "globset" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a1e17342619edbc21a964c2afbeb6c820c6a2560032872f397bb97ea127bd0a" +checksum = "029d74589adefde59de1a0c4f4732695c32805624aec7b68d91503d4dba79afc" dependencies = [ "aho-corasick", - "bstr 0.2.17", + "bstr 1.4.0", "fnv", "log", "regex", @@ -340,13 +376,18 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "ignore" -version = "0.4.18" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713f1b139373f96a2e0ce3ac931cd01ee973c3c5dd7c40c0c2efe96ad2b6751d" +checksum = "dbe7873dab538a9a44ad79ede1faf5f30d49f9a5c883ddbab48bce81b64b7492" dependencies = [ - "crossbeam-utils", "globset", "lazy_static", "log", @@ -358,6 +399,26 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "io-lifetimes" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +dependencies = [ + "hermit-abi 0.3.1", + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -396,9 +457,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.137" +version = "0.2.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" +checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" [[package]] name = "libloading" @@ -410,6 +471,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "linux-raw-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece97ea872ece730aed82664c424eb4c8291e1ff2480247ccf7409044bc6479f" + [[package]] name = "log" version = "0.4.17" @@ -523,7 +590,7 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", ] @@ -650,6 +717,15 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.7.0" @@ -676,6 +752,20 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +[[package]] +name = "rustix" +version = "0.37.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes", + "libc", + "linux-raw-sys", + "windows-sys 0.48.0", +] + [[package]] name = "ryu" version = "1.0.11" @@ -786,15 +876,18 @@ version = "0.1.0" name = "tailwindcss-core" version = "0.1.0" dependencies = [ - "bstr 1.0.1", + "bstr 1.4.0", "criterion", "crossbeam", "fxhash", "globwalk", + "ignore", "log", "rayon", + "tempfile", "tracing", "tracing-subscriber", + "walkdir", ] [[package]] @@ -806,6 +899,19 @@ dependencies = [ "tailwindcss-core", ] +[[package]] +name = "tempfile" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys 0.45.0", +] + [[package]] name = "termcolor" version = "1.1.3" @@ -937,12 +1043,11 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" dependencies = [ "same-file", - "winapi", "winapi-util", ] @@ -1040,3 +1145,135 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" diff --git a/oxide/crates/core/Cargo.toml b/oxide/crates/core/Cargo.toml index ae895f34a16e..0db8eb2b214d 100644 --- a/oxide/crates/core/Cargo.toml +++ b/oxide/crates/core/Cargo.toml @@ -12,9 +12,12 @@ fxhash = "0.2.1" crossbeam = "0.8.2" tracing = { version = "0.1.37", features = [] } tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } +walkdir = "2.3.3" +ignore = "0.4.20" [dev-dependencies] criterion = { version = "0.3", features = ['html_reports'] } +tempfile = "3.5.0" [[bench]] name = "parse_candidates" diff --git a/oxide/crates/core/src/fixtures/binary-extensions.txt b/oxide/crates/core/src/fixtures/binary-extensions.txt new file mode 100644 index 000000000000..b7cee312ceee --- /dev/null +++ b/oxide/crates/core/src/fixtures/binary-extensions.txt @@ -0,0 +1,259 @@ +3dm +3ds +3g2 +3gp +7z +DS_Store +a +aac +adp +ai +aif +aiff +alz +ape +apk +appimage +ar +arj +asf +au +avi +bak +baml +bh +bin +bk +bmp +btif +bz2 +bzip2 +cab +caf +cgm +class +cmx +cpio +cr2 +cur +dat +dcm +deb +dex +djvu +dll +dmg +dng +doc +docm +docx +dot +dotm +dra +dsk +dts +dtshd +dvb +dwg +dxf +ecelp4800 +ecelp7470 +ecelp9600 +egg +eol +eot +epub +exe +f4v +fbs +fh +fla +flac +flatpak +fli +flv +fpx +fst +fvt +g3 +gh +gif +graffle +gz +gzip +h261 +h263 +h264 +icns +ico +ief +img +ipa +iso +jar +jpeg +jpg +jpgv +jpm +jxr +key +ktx +lha +lib +lockb +lvp +lz +lzh +lzma +lzo +m3u +m4a +m4v +mar +mdi +mht +mid +midi +mj2 +mka +mkv +mmr +mng +mobi +mov +movie +mp3 +mp4 +mp4a +mpeg +mpg +mpga +mxu +nef +npx +numbers +nupkg +o +odp +ods +odt +oga +ogg +ogv +otf +ott +pages +pbm +pcx +pdb +pdf +pea +pgm +pic +png +pnm +pot +potm +potx +ppa +ppam +ppm +pps +ppsm +ppsx +ppt +pptm +pptx +psd +pya +pyc +pyo +pyv +qt +rar +ras +raw +resources +rgb +rip +rlc +rmf +rmvb +rpm +rtf +rz +s3m +s7z +scpt +sgi +shar +sil +sketch +slk +smv +snap +snk +so +stl +sub +suo +swf +tar +tbz +tbz2 +tga +tgz +thmx +tif +tiff +tlz +ttc +ttf +txz +udf +uvh +uvi +uvm +uvp +uvs +uvu +viv +vob +war +wav +wax +wbmp +wdp +weba +webm +webp +whl +wim +wm +wma +wmv +wmx +woff +woff2 +wrm +wvx +xbm +xif +xla +xlam +xls +xlsb +xlsm +xlsx +xlt +xltm +xltx +xm +xmind +xpi +xpm +xwd +xz +z +zip +zipx diff --git a/oxide/crates/core/src/fixtures/ignored-extensions.txt b/oxide/crates/core/src/fixtures/ignored-extensions.txt new file mode 100644 index 000000000000..f147c24fe110 --- /dev/null +++ b/oxide/crates/core/src/fixtures/ignored-extensions.txt @@ -0,0 +1,6 @@ +css +less +lock +sass +scss +styl diff --git a/oxide/crates/core/src/fixtures/ignored-files.txt b/oxide/crates/core/src/fixtures/ignored-files.txt new file mode 100644 index 000000000000..45d4ced87afd --- /dev/null +++ b/oxide/crates/core/src/fixtures/ignored-files.txt @@ -0,0 +1,3 @@ +package-lock.json +pnpm-lock.yaml +bun.lockb diff --git a/oxide/crates/core/src/fixtures/template-extensions.txt b/oxide/crates/core/src/fixtures/template-extensions.txt new file mode 100644 index 000000000000..c059eff3a9ed --- /dev/null +++ b/oxide/crates/core/src/fixtures/template-extensions.txt @@ -0,0 +1,57 @@ +# HTML +html +pug + +# JS +astro +cjs +cts +jade +js +jsx +mjs +mts +svelte +ts +tsx +vue + +# Markdown +md +mdx + +# ASP +aspx +razor + +# Handlebars +handlebars +hbs +mustache + +# PHP +php +twig + +# Ruby +erb +haml +liquid +rb +rhtml +slim + +# Elixir / Phoenix +eex +heex + +# Nunjucks +njk +nunjucks + +# Python +py +tpl + +# Rust +rs diff --git a/oxide/crates/core/src/lib.rs b/oxide/crates/core/src/lib.rs index 9dd88437dfdd..6601267fd27c 100644 --- a/oxide/crates/core/src/lib.rs +++ b/oxide/crates/core/src/lib.rs @@ -1,8 +1,12 @@ use crate::parser::Extractor; use fxhash::FxHashSet; +use ignore::WalkBuilder; use rayon::prelude::*; +use std::cmp::Ordering; +use std::path::Path; use std::path::PathBuf; use tracing::event; +use walkdir::WalkDir; pub mod candidate; pub mod glob; @@ -30,6 +34,275 @@ pub struct ChangedContent { pub extension: String, } +#[derive(Debug, Clone)] +pub struct ContentPathInfo { + pub base: String, +} + +pub fn resolve_content_paths(args: ContentPathInfo) -> Vec { + let root = Path::new(&args.base); + + let allowed_paths = FxHashSet::from_iter( + WalkBuilder::new(&root) + .hidden(false) + .filter_entry(|entry| match entry.file_type() { + Some(file_type) if file_type.is_dir() => entry + .file_name() + .to_str() + .map(|s| s != ".git") + .unwrap_or(false), + _ => true, + }) + .build() + .filter_map(Result::ok) + .map(|x| x.into_path()), + ); + + // A list of directory names where we can't use globs, but we should track each file + // individually instead. This is because these directories are often used for both source and + // destination files. + let mut forced_static_directories = vec![root.join("public")]; + + // A list of known extensions + a list of extensions we found in the project. + let mut found_extensions = FxHashSet::from_iter( + include_str!("fixtures/template-extensions.txt") + .trim() + .lines() + .filter(|x| !x.starts_with('#')) // Drop commented lines + .filter(|x| !x.is_empty()) // Drop empty lines + .map(|x| x.to_string()), + ); + + // A list of static files that we should track directly. + let mut static_files = vec![]; + + // All root directories. + let mut root_directories = FxHashSet::from_iter(vec![root.to_path_buf()]); + + // All directories where we can safely use deeply nested globs to watch all files. + // In other comments we refer to these as "deep glob directories" or similar. + // + // E.g.: ./src/**/*.{html,js} + let mut deep_globable_directories: FxHashSet = FxHashSet::default(); + + // All directories where we can only use shallow globs to watch all direct files but not + // folders. + // In other comments we refer to these as "shallow glob directories" or similar. + // + // E.g.: ./src/*/*.{html,js} + let mut shallow_globable_directories: FxHashSet = FxHashSet::default(); + + // Collect all valid paths from the root. This will already filter out ignored files, unknown + // extensions and binary files. + let mut it = WalkDir::new(&root) + // Sorting to make sure that we always see the directories before the files. Also sorting + // alphabetically by default. + .sort_by( + |a, z| match (a.file_type().is_dir(), z.file_type().is_dir()) { + (true, false) => Ordering::Less, + (false, true) => Ordering::Greater, + _ => a.file_name().cmp(z.file_name()), + }, + ) + .into_iter(); + + loop { + // We are only interested in valid entries + let entry = match it.next() { + Some(Ok(entry)) => entry, + _ => break, + }; + + // Ignore known directories that we don't want to traverse into. + if entry.file_type().is_dir() && entry.file_name() == ".git" { + it.skip_current_dir(); + continue; + } + + if entry.file_type().is_dir() { + // If we are in a directory where we know that we can't use any globs, then we have to + // track each file individually. + if forced_static_directories.contains(&entry.path().to_path_buf()) { + forced_static_directories.push(entry.path().to_path_buf()); + root_directories.insert(entry.path().to_path_buf()); + continue; + } + + // If we are in a directory where the parent is a forced static directory, then this + // will become a forced static directory as well. + if forced_static_directories.contains(&entry.path().parent().unwrap().to_path_buf()) { + forced_static_directories.push(entry.path().to_path_buf()); + root_directories.insert(entry.path().to_path_buf()); + continue; + } + + // If we are in a directory, and the directory is git ignored, then we don't have to + // descent into the directory. However, we have to make sure that we mark the _parent_ + // directory as a shallow glob directory because using deep globs from any of the + // parent directories will include this ignored directory which should not be the case. + // + // Another important part is that if one of the ignored directories is a deep glob + // directory, then all of its parents (until the root) should be marked as shallow glob + // directories as well. + if !allowed_paths.contains(&entry.path().to_path_buf()) { + let mut parent = entry.path().parent(); + while let Some(parent_path) = parent { + // If the parent is already marked as a valid deep glob directory, then we have + // to mark it as a shallow glob directory instead, because we won't be able to + // use deep globs for this directory anymore. + if deep_globable_directories.contains(parent_path) { + deep_globable_directories.remove(parent_path); + shallow_globable_directories.insert(parent_path.to_path_buf()); + } + + // If we reached the root, then we can stop. + if parent_path == root { + break; + } + + // Mark the parent directory as a shallow glob directory and continue with its + // parent. + shallow_globable_directories.insert(parent_path.to_path_buf()); + parent = parent_path.parent(); + } + + it.skip_current_dir(); + continue; + } + // If we are in a directory that is not git ignored, then we can mark this directory as + // a valid deep glob directory. This is only necessary if any of its parents aren't + // marked as deep glob directories already. + else { + let mut found_deep_glob_parent = false; + let mut parent = entry.path().parent(); + while let Some(parent_path) = parent { + // If we reached the root, then we can stop. + if parent_path == root { + break; + } + + // If the parent is already marked as a deep glob directory, then we can stop + // because this glob will match the current directory already. + if deep_globable_directories.contains(parent_path) { + found_deep_glob_parent = true; + break; + } + + parent = parent_path.parent(); + } + + // If we didn't find a deep glob directory parent, then we can mark this directory + // as a deep glob directory (unless it is the root). + if !found_deep_glob_parent && entry.path() != root { + deep_globable_directories.insert(entry.path().to_path_buf()); + } + } + } + + // Handle allowed content paths + if is_allowed_content_path(entry.path()) + && allowed_paths.contains(&entry.path().to_path_buf()) + { + let path = entry.path(); + + // Collect the extension for future use when building globs. + if let Some(extension) = path.extension().and_then(|x| x.to_str()) { + found_extensions.insert(extension.to_string()); + } + + // If the current path is a file inside any of the root directories, then we have to + // track it directly as a static file. + if let Some(parent_path) = path.parent() { + if root_directories.contains(parent_path) { + static_files.push(format!("{}", path.display())); + } + } + } + } + + // Build the globs for all globable directories. + let shallow_globs = shallow_globable_directories + .iter() + .map(|path| format!("{}", path.display())); + let deep_globs = deep_globable_directories + .iter() + .map(|path| format!("{}/**", path.display())); + let globs = shallow_globs + .chain(deep_globs) + .flat_map(|path| match found_extensions.len() { + 0 => None, // This should never happen + 1 => Some(format!( + "{}/*.{}", + path, + found_extensions.iter().next().unwrap() + )), + _ => Some(format!( + "{}/*.{{{}}}", + path, + found_extensions + .iter() + .map(|x| x.to_string()) + .collect::>() + .join(","), + )), + }) + .collect::>(); + + static_files.extend(globs); + static_files +} + +pub fn is_git_ignored_content_path(base: &Path, path: &Path) -> bool { + !WalkBuilder::new(base) + .hidden(false) + .filter_entry(|entry| match entry.file_type() { + Some(file_type) if file_type.is_dir() => entry + .file_name() + .to_str() + .map(|s| s != ".git") + .unwrap_or(false), + _ => true, + }) + .build() + .filter_map(Result::ok) + .any(|e| e.path() == path) +} + +pub fn is_allowed_content_path(path: &Path) -> bool { + let binary_extensions = include_str!("fixtures/binary-extensions.txt") + .trim() + .lines() + .collect::>(); + let ignored_extensions = include_str!("fixtures/ignored-extensions.txt") + .trim() + .lines() + .collect::>(); + let ignored_files = include_str!("fixtures/ignored-files.txt") + .trim() + .lines() + .collect::>(); + + let path = PathBuf::from(path); + + // Skip known ignored files + if path + .file_name() + .unwrap() + .to_str() + .map(|s| ignored_files.contains(&s)) + .unwrap_or(false) + { + return false; + } + + // Skip known ignored extensions + return path + .extension() + .map(|s| s.to_str().unwrap_or_default()) + .map(|ext| !ignored_extensions.contains(&ext) && !binary_extensions.contains(&ext)) + .unwrap_or(false); +} + #[derive(Debug)] pub enum IO { Sequential = 0b0001, @@ -89,7 +362,13 @@ fn read_all_files(changed_content: Vec) -> Vec> { changed_content .into_par_iter() .map(|c| match (c.file, c.content) { - (Some(file), None) => std::fs::read(file).unwrap(), + (Some(file), None) => match std::fs::read(file) { + Ok(content) => content, + Err(e) => { + event!(tracing::Level::ERROR, "Failed to read file: {:?}", e); + Default::default() + } + }, (None, Some(content)) => content.into_bytes(), _ => Default::default(), }) diff --git a/oxide/crates/core/src/parser.rs b/oxide/crates/core/src/parser.rs index 16831afffd41..372758e8ebff 100644 --- a/oxide/crates/core/src/parser.rs +++ b/oxide/crates/core/src/parser.rs @@ -352,7 +352,7 @@ impl<'a> Extractor<'a> { self.in_candidate && !self.in_arbitrary && (0..=127).contains(&c) - && (self.idx_start == 0 || self.input[(self.idx_start - 1)] <= 127) + && (self.idx_start == 0 || self.input[self.idx_start - 1] <= 127) } #[inline(always)] @@ -470,7 +470,7 @@ impl<'a> Iterator for Extractor<'a> { mod test { use super::*; - fn please_trace() { + fn _please_trace() { tracing_subscriber::fmt() .with_max_level(tracing::Level::TRACE) .with_span_events(tracing_subscriber::fmt::format::FmtSpan::ACTIVE) diff --git a/oxide/crates/core/tests/auto_content.rs b/oxide/crates/core/tests/auto_content.rs new file mode 100644 index 000000000000..f7a185f6ce36 --- /dev/null +++ b/oxide/crates/core/tests/auto_content.rs @@ -0,0 +1,248 @@ +#[cfg(test)] +mod auto_content { + use std::fs; + use std::process::Command; + + use tailwindcss_core::*; + use tempfile::tempdir; + + fn test(paths_with_content: &[(&str, Option<&str>)]) -> Vec { + // Create a temporary working directory + let dir = tempdir().unwrap().into_path(); + + // Initialize this directory as a git repository + let _ = Command::new("git").arg("init").current_dir(&dir).output(); + + // Create the necessary files + for (path, contents) in paths_with_content { + let path = dir.join(path); + let parent = path.parent().unwrap(); + if !parent.exists() { + fs::create_dir_all(parent).unwrap(); + } + + match contents { + Some(contents) => fs::write(path, contents).unwrap(), + None => fs::write(path, "").unwrap(), + } + } + + let base = format!("{}", dir.display()); + + // Resolve all content paths for the (temporary) current working directory + let mut paths: Vec<_> = resolve_content_paths(ContentPathInfo { base: base.clone() }) + .into_iter() + .map(|x| x.replace(&format!("{}/", &base), "")) + .collect(); + + // Sort the output for easier comparison (depending on internal datastructure the order + // _could_ be random) + paths.sort(); + + paths + } + + #[test] + fn it_should_work_with_a_set_of_root_files() { + let globs = test(&[ + ("index.html", None), + ("a.html", None), + ("b.html", None), + ("c.html", None), + ]); + assert_eq!(globs, vec!["a.html", "b.html", "c.html", "index.html"]); + } + + #[test] + fn it_should_work_with_a_set_of_root_files_and_ignore_ignored_files() { + let globs = test(&[ + (".gitignore", Some("b.html")), + ("index.html", None), + ("a.html", None), + ("b.html", None), + ("c.html", None), + ]); + assert_eq!(globs, vec!["a.html", "c.html", "index.html"]); + } + + #[test] + fn it_should_list_all_files_in_the_public_folder_explicitly() { + let globs = test(&[ + ("index.html", None), + ("public/a.html", None), + ("public/b.html", None), + ("public/c.html", None), + ]); + assert_eq!( + globs, + vec![ + "index.html", + "public/a.html", + "public/b.html", + "public/c.html", + ] + ); + } + + #[test] + fn it_should_list_nested_folders_explicitly_in_the_public_folder() { + let globs = test(&[ + ("index.html", None), + ("public/a.html", None), + ("public/b.html", None), + ("public/c.html", None), + ("public/nested/a.html", None), + ("public/nested/b.html", None), + ("public/nested/c.html", None), + ("public/nested/again/a.html", None), + ("public/very/deeply/nested/a.html", None), + ]); + assert_eq!( + globs, + vec![ + "index.html", + "public/a.html", + "public/b.html", + "public/c.html", + "public/nested/a.html", + "public/nested/again/a.html", + "public/nested/b.html", + "public/nested/c.html", + "public/very/deeply/nested/a.html", + ] + ); + } + + #[test] + fn it_should_list_all_files_in_the_public_folder_explicitly_except_ignored_files() { + let globs = test(&[ + (".gitignore", Some("public/b.html\na.html")), + ("index.html", None), + ("public/a.html", None), + ("public/b.html", None), + ("public/c.html", None), + ]); + assert_eq!(globs, vec!["index.html", "public/c.html",]); + } + + #[test] + fn it_should_use_a_glob_for_top_level_folders() { + let globs = test(&[ + ("index.html", None), + ("src/a.html", None), + ("src/b.html", None), + ("src/c.html", None), + ]); + assert_eq!(globs, vec!["index.html", "src/**/*.{py,tpl,js,vue,php,mjs,cts,jsx,tsx,rhtml,slim,handlebars,twig,rs,njk,svelte,liquid,pug,md,ts,heex,mts,astro,nunjucks,rb,eex,haml,cjs,html,hbs,jade,aspx,razor,erb,mustache,mdx}"]); + } + + #[test] + fn it_should_ignore_binary_files() { + let globs = test(&[ + ("index.html", None), + ("a.mp4", None), + ("b.png", None), + ("c.lock", None), + ]); + assert_eq!(globs, vec!["index.html"]); + } + + #[test] + fn it_should_ignore_known_extensions() { + let globs = test(&[ + ("index.html", None), + ("a.css", None), + ("b.sass", None), + ("c.less", None), + ]); + assert_eq!(globs, vec!["index.html"]); + } + + #[test] + fn it_should_ignore_known_files() { + let globs = test(&[ + ("index.html", None), + ("package-lock.json", None), + ("yarn.lock", None), + ]); + assert_eq!(globs, vec!["index.html"]); + } + + #[test] + fn it_should_ignore_and_expand_nested_ignored_folders() { + let globs = test(&[ + // Explicitly listed root files + ("foo.html", None), + ("bar.html", None), + ("baz.html", None), + // Nested folder A, using glob + ("nested-a/foo.html", None), + ("nested-a/bar.html", None), + ("nested-a/baz.html", None), + // Nested folder B, with deeply nested files, using glob + ("nested-b/deeply-nested/foo.html", None), + ("nested-b/deeply-nested/bar.html", None), + ("nested-b/deeply-nested/baz.html", None), + // Nested folder C, with ignored sub-folder + ("nested-c/foo.html", None), + ("nested-c/bar.html", None), + ("nested-c/baz.html", None), + // Ignored folder + ("nested-c/.gitignore", Some("ignored-folder/")), + ("nested-c/ignored-folder/foo.html", None), + ("nested-c/ignored-folder/bar.html", None), + ("nested-c/ignored-folder/baz.html", None), + // Deeply nested, without issues + ("nested-c/sibling-folder/foo.html", None), + ("nested-c/sibling-folder/bar.html", None), + ("nested-c/sibling-folder/baz.html", None), + // Nested folder D, with deeply nested ignored folder + ("nested-d/foo.html", None), + ("nested-d/bar.html", None), + ("nested-d/baz.html", None), + ("nested-d/.gitignore", Some("deep/")), + ("nested-d/very/deeply/nested/deep/foo.html", None), + ("nested-d/very/deeply/nested/deep/bar.html", None), + ("nested-d/very/deeply/nested/deep/baz.html", None), + ("nested-d/very/deeply/nested/foo.html", None), + ("nested-d/very/deeply/nested/bar.html", None), + ("nested-d/very/deeply/nested/baz.html", None), + ("nested-d/very/deeply/nested/directory/foo.html", None), + ("nested-d/very/deeply/nested/directory/bar.html", None), + ("nested-d/very/deeply/nested/directory/baz.html", None), + ("nested-d/very/deeply/nested/directory/again/foo.html", None), + ]); + + assert_eq!( + globs, + vec![ + // Listed explicitly because they are in the root + "bar.html", + "baz.html", + "foo.html", + + // Listed using a deep glob, because nothing inside is ignored + "nested-a/**/*.{py,tpl,js,vue,php,mjs,cts,jsx,tsx,rhtml,slim,handlebars,twig,rs,njk,svelte,liquid,pug,md,ts,heex,mts,astro,nunjucks,rb,eex,haml,cjs,html,hbs,jade,aspx,razor,erb,mustache,mdx}", + + // Listed using a deep glob, because nothing inside is ignored (but contains deeply + // nested folders) + "nested-b/**/*.{py,tpl,js,vue,php,mjs,cts,jsx,tsx,rhtml,slim,handlebars,twig,rs,njk,svelte,liquid,pug,md,ts,heex,mts,astro,nunjucks,rb,eex,haml,cjs,html,hbs,jade,aspx,razor,erb,mustache,mdx}", + + // Listed using a shallow glob, because `nested-c` contains ignored folders, deeply + // nested globs can't be used anymore + "nested-c/*.{py,tpl,js,vue,php,mjs,cts,jsx,tsx,rhtml,slim,handlebars,twig,rs,njk,svelte,liquid,pug,md,ts,heex,mts,astro,nunjucks,rb,eex,haml,cjs,html,hbs,jade,aspx,razor,erb,mustache,mdx}", + + // Deeply nested folder can use a deep glob again + "nested-c/sibling-folder/**/*.{py,tpl,js,vue,php,mjs,cts,jsx,tsx,rhtml,slim,handlebars,twig,rs,njk,svelte,liquid,pug,md,ts,heex,mts,astro,nunjucks,rb,eex,haml,cjs,html,hbs,jade,aspx,razor,erb,mustache,mdx}", + + // The deeply nested folder can use a deep glob, however all of its parents can + // only contain a shallow glob because of the ignored folder + "nested-d/*.{py,tpl,js,vue,php,mjs,cts,jsx,tsx,rhtml,slim,handlebars,twig,rs,njk,svelte,liquid,pug,md,ts,heex,mts,astro,nunjucks,rb,eex,haml,cjs,html,hbs,jade,aspx,razor,erb,mustache,mdx}", + "nested-d/very/*.{py,tpl,js,vue,php,mjs,cts,jsx,tsx,rhtml,slim,handlebars,twig,rs,njk,svelte,liquid,pug,md,ts,heex,mts,astro,nunjucks,rb,eex,haml,cjs,html,hbs,jade,aspx,razor,erb,mustache,mdx}", + "nested-d/very/deeply/*.{py,tpl,js,vue,php,mjs,cts,jsx,tsx,rhtml,slim,handlebars,twig,rs,njk,svelte,liquid,pug,md,ts,heex,mts,astro,nunjucks,rb,eex,haml,cjs,html,hbs,jade,aspx,razor,erb,mustache,mdx}", + "nested-d/very/deeply/nested/*.{py,tpl,js,vue,php,mjs,cts,jsx,tsx,rhtml,slim,handlebars,twig,rs,njk,svelte,liquid,pug,md,ts,heex,mts,astro,nunjucks,rb,eex,haml,cjs,html,hbs,jade,aspx,razor,erb,mustache,mdx}", + "nested-d/very/deeply/nested/directory/**/*.{py,tpl,js,vue,php,mjs,cts,jsx,tsx,rhtml,slim,handlebars,twig,rs,njk,svelte,liquid,pug,md,ts,heex,mts,astro,nunjucks,rb,eex,haml,cjs,html,hbs,jade,aspx,razor,erb,mustache,mdx}", + ] + ); + } +} diff --git a/oxide/crates/node/package.json b/oxide/crates/node/package.json index f9841b69f648..80ddc6fcffed 100644 --- a/oxide/crates/node/package.json +++ b/oxide/crates/node/package.json @@ -19,7 +19,7 @@ }, "license": "MIT", "devDependencies": { - "@napi-rs/cli": "^2.15.2" + "@napi-rs/cli": "2.15.2" }, "engines": { "node": ">= 10" diff --git a/oxide/crates/node/src/lib.rs b/oxide/crates/node/src/lib.rs index f8b927db5a60..82bf28b0acc1 100644 --- a/oxide/crates/node/src/lib.rs +++ b/oxide/crates/node/src/lib.rs @@ -29,6 +29,17 @@ pub fn parse_candidate_strings_from_files(changed_content: Vec) ) } +#[derive(Debug, Clone)] +#[napi(object)] +pub struct ContentPathInfo { + pub base: String, +} + +#[napi] +pub fn resolve_content_paths(args: ContentPathInfo) -> Vec { + tailwindcss_core::resolve_content_paths(tailwindcss_core::ContentPathInfo { base: args.base }) +} + #[derive(Debug)] #[napi] pub enum IO { diff --git a/package-lock.json b/package-lock.json index bf0740111ded..24edfdd2f0cf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16472,7 +16472,7 @@ "version": "0.0.0", "license": "MIT", "devDependencies": { - "@napi-rs/cli": "^2.15.2" + "@napi-rs/cli": "2.15.2" }, "engines": { "node": ">= 10" @@ -19253,7 +19253,7 @@ "@tailwindcss/oxide": { "version": "file:oxide/crates/node", "requires": { - "@napi-rs/cli": "^2.15.2", + "@napi-rs/cli": "2.15.2", "@tailwindcss/oxide-darwin-arm64": "0.0.0", "@tailwindcss/oxide-darwin-x64": "0.0.0", "@tailwindcss/oxide-freebsd-x64": "0.0.0", @@ -29917,7 +29917,7 @@ "@tailwindcss/oxide": { "version": "file:oxide/crates/node", "requires": { - "@napi-rs/cli": "^2.15.2", + "@napi-rs/cli": "2.15.2", "@tailwindcss/oxide-darwin-arm64": "0.0.0", "@tailwindcss/oxide-darwin-x64": "0.0.0", "@tailwindcss/oxide-freebsd-x64": "0.0.0", diff --git a/src/cli/build/plugin.js b/src/cli/build/plugin.js index 9f3f6f52ff27..a5045593d13e 100644 --- a/src/cli/build/plugin.js +++ b/src/cli/build/plugin.js @@ -19,6 +19,7 @@ import { findAtConfigPath } from '../../lib/findAtConfigPath.js' import log from '../../util/log' import { loadConfig } from '../../lib/load-config' import getModuleDependencies from '../../lib/getModuleDependencies' +import { validateConfig } from '../../util/validateConfig' /** * @@ -161,7 +162,13 @@ let state = { } // @ts-ignore - this.configBag.config = resolveConfig(this.configBag.config, { content: { files: [] } }) + if (__OXIDE__) { + this.configBag.config = validateConfig(resolveConfig(this.configBag.config)) + } else { + this.configBag.config = validateConfig( + resolveConfig(this.configBag.config, { content: { files: [] } }) + ) + } // Override content files if `--content` has been passed explicitly if (content?.length > 0) { diff --git a/src/cli/init/index.js b/src/cli/init/index.js index 6bd7e41e2ced..ac118549adf1 100644 --- a/src/cli/init/index.js +++ b/src/cli/init/index.js @@ -38,6 +38,11 @@ export function init(args) { 'utf8' ) + // Drop `content` in the oxide engine to promote auto content + if (__OXIDE__) { + stubContentsFile = stubContentsFile.replace(/\s*content: \[\],\n/, '') + } + // Change colors import stubContentsFile = stubContentsFile.replace('../colors', 'tailwindcss/colors') diff --git a/src/lib/content.js b/src/lib/content.js index e814efe4d34e..7b58e9d67b7b 100644 --- a/src/lib/content.js +++ b/src/lib/content.js @@ -11,6 +11,28 @@ import { env } from './sharedState' /** @typedef {import('../../types/config.js').RawFile} RawFile */ /** @typedef {import('../../types/config.js').FilePath} FilePath */ +/* + * @param {import('tailwindcss').Config} tailwindConfig + * @param {{skip:string[]}} options + * @returns {ContentPath[]} + */ +function resolveContentFiles(tailwindConfig, { skip = [] } = {}) { + if (tailwindConfig.content.files === 'auto' && __OXIDE__) { + env.DEBUG && console.time('Calculating resolve content paths') + tailwindConfig.content.files = require('@tailwindcss/oxide').resolveContentPaths({ + base: process.cwd(), + }) + if (skip.length > 0) { + tailwindConfig.content.files = tailwindConfig.content.files.filter( + (filePath) => !skip.includes(filePath) + ) + } + env.DEBUG && console.timeEnd('Calculating resolve content paths') + } + + return tailwindConfig.content.files +} + /** * @typedef {object} ContentPath * @property {string} original @@ -32,7 +54,9 @@ import { env } from './sharedState' * @returns {ContentPath[]} */ export function parseCandidateFiles(context, tailwindConfig) { - let files = tailwindConfig.content.files + let files = resolveContentFiles(tailwindConfig, { + skip: [context.userConfigPath], + }) // Normalize the file globs files = files.filter((filePath) => typeof filePath === 'string') @@ -167,9 +191,12 @@ function resolvePathSymlinks(contentPath) { * @returns {[{ content: string, extension: string }[], Map]} */ export function resolvedChangedContent(context, candidateFiles, fileModifiedMap) { - let changedContent = context.tailwindConfig.content.files - .filter((item) => typeof item.raw === 'string') - .map(({ raw, extension = 'html' }) => ({ content: raw, extension })) + let changedContent = + context.tailwindConfig.content.files === 'auto' && __OXIDE__ + ? [] + : context.tailwindConfig.content.files + .filter((item) => typeof item.raw === 'string') + .map(({ raw, extension = 'html' }) => ({ content: raw, extension })) let [changedFiles, mTimesToCommit] = resolveChangedFiles(candidateFiles, fileModifiedMap) diff --git a/src/oxide/cli/build/plugin.ts b/src/oxide/cli/build/plugin.ts index 4141ec439b0e..aa9bb5b99e6c 100644 --- a/src/oxide/cli/build/plugin.ts +++ b/src/oxide/cli/build/plugin.ts @@ -18,6 +18,7 @@ import log from '../../../util/log' import { loadConfig } from '../../../lib/load-config' import getModuleDependencies from '../../../lib/getModuleDependencies' import type { Config } from '../../../../types' +import { validateConfig } from '../../../util/validateConfig' /** * @@ -160,7 +161,13 @@ let state = { } // @ts-ignore - this.configBag.config = resolveConfig(this.configBag.config, { content: { files: [] } }) + if (__OXIDE__) { + this.configBag.config = validateConfig(resolveConfig(this.configBag.config)) + } else { + this.configBag.config = validateConfig( + resolveConfig(this.configBag.config, { content: { files: [] } }) + ) + } // Override content files if `--content` has been passed explicitly if (content?.length > 0) { diff --git a/src/oxide/cli/init/index.ts b/src/oxide/cli/init/index.ts index abc93cdc1351..b05138349b5f 100644 --- a/src/oxide/cli/init/index.ts +++ b/src/oxide/cli/init/index.ts @@ -36,6 +36,11 @@ export function init(args) { 'utf8' ) + // Drop `content` in the oxide engine to promote auto content + if (__OXIDE__) { + stubContentsFile = stubContentsFile.replace(/\s*content: \[\],\n/, '') + } + // Change colors import stubContentsFile = stubContentsFile.replace('../colors', 'tailwindcss/colors') diff --git a/src/util/getAllConfigs.js b/src/util/getAllConfigs.js index ce3665b977d0..90a20e11a251 100644 --- a/src/util/getAllConfigs.js +++ b/src/util/getAllConfigs.js @@ -2,7 +2,14 @@ import defaultFullConfig from '../../stubs/config.full.js' import { flagEnabled } from '../featureFlags' export default function getAllConfigs(config) { - const configs = (config?.presets ?? [defaultFullConfig]) + const configs = ( + config?.presets ?? [ + __OXIDE__ + ? // Drop `content` in the oxide engine to promote auto content + Object.assign({}, defaultFullConfig, { content: 'auto' }) + : defaultFullConfig, + ] + ) .slice() .reverse() .flatMap((preset) => getAllConfigs(preset instanceof Function ? preset() : preset)) diff --git a/src/util/normalizeConfig.js b/src/util/normalizeConfig.js index 7e1a592decda..a79aaefd0660 100644 --- a/src/util/normalizeConfig.js +++ b/src/util/normalizeConfig.js @@ -18,6 +18,10 @@ export function normalizeConfig(config) { * } */ let valid = (() => { + if (config.content === 'auto') { + return true + } + // `config.purge` should not exist anymore if (config.purge) { return false @@ -181,6 +185,31 @@ export function normalizeConfig(config) { config.prefix = config.prefix ?? '' } + let auto = (() => { + // Config still has a `purge` option (for backwards compatibility), auto content should not be + // used + if (config.purge) return false + + // + if (config.content === 'auto') return true + + // We don't have content at all, auto content should be used + if (config.content === undefined) return true + + // We do have content as an object, but we don't have any files defined, auto content should + // be used + if ( + typeof config.content === 'object' && + config.content !== null && + !Array.isArray(config.content) + ) { + return config.content.files === undefined + } + + // We do have content defined, auto content should not be used + return false + })() + // Normalize the `content` config.content = { relative: (() => { @@ -193,17 +222,20 @@ export function normalizeConfig(config) { return flagEnabled(config, 'relativeContentPathsByDefault') })(), - files: (() => { - let { content, purge } = config + files: auto + ? 'auto' + : (() => { + let { content, purge } = config - if (Array.isArray(purge)) return purge - if (Array.isArray(purge?.content)) return purge.content - if (Array.isArray(content)) return content - if (Array.isArray(content?.content)) return content.content - if (Array.isArray(content?.files)) return content.files + if (content === undefined && purge === undefined) return [] + if (Array.isArray(purge)) return purge + if (Array.isArray(purge?.content)) return purge.content + if (Array.isArray(content)) return content + if (Array.isArray(content?.content)) return content.content + if (Array.isArray(content?.files)) return content.files - return [] - })(), + return [] + })(), extract: (() => { let extract = (() => { @@ -286,14 +318,16 @@ export function normalizeConfig(config) { // Validate globs to prevent bogus globs. // E.g.: `./src/*.{html}` is invalid, the `{html}` should just be `html` - for (let file of config.content.files) { - if (typeof file === 'string' && /{([^,]*?)}/g.test(file)) { - log.warn('invalid-glob-braces', [ - `The glob pattern ${dim(file)} in your Tailwind CSS configuration is invalid.`, - `Update it to ${dim(file.replace(/{([^,]*?)}/g, '$1'))} to silence this warning.`, - // TODO: Add https://tw.wtf/invalid-glob-braces - ]) - break + if (config.content.files !== 'auto') { + for (let file of config.content.files) { + if (typeof file === 'string' && /{([^,]*?)}/g.test(file)) { + log.warn('invalid-glob-braces', [ + `The glob pattern ${dim(file)} in your Tailwind CSS configuration is invalid.`, + `Update it to ${dim(file.replace(/{([^,]*?)}/g, '$1'))} to silence this warning.`, + // TODO: Add https://tw.wtf/invalid-glob-braces + ]) + break + } } } diff --git a/src/util/validateConfig.js b/src/util/validateConfig.js index 8c22e445035b..dfba0184469c 100644 --- a/src/util/validateConfig.js +++ b/src/util/validateConfig.js @@ -1,7 +1,7 @@ import log from './log' export function validateConfig(config) { - if (config.content.files.length === 0) { + if (config.content.files !== 'auto' && config.content.files.length === 0) { log.warn('content-problems', [ 'The `content` option in your Tailwind CSS configuration is missing or empty.', 'Configure your content sources or your generated CSS will be missing styles.', @@ -9,6 +9,13 @@ export function validateConfig(config) { ]) } + if (config.content.files === 'auto') { + log.warn('auto-content-experimental', [ + 'Automatic content detection in Tailwind CSS is currently in experimental preview.', + 'Preview features are not covered by semver, and may change at any time.', + ]) + } + // Warn if the line-clamp plugin is installed try { let plugin = require('@tailwindcss/line-clamp') diff --git a/tests/normalize-config.test.js b/tests/normalize-config.test.js index a0b7a59bb950..47980cbdde72 100644 --- a/tests/normalize-config.test.js +++ b/tests/normalize-config.test.js @@ -7,7 +7,7 @@ crosscheck(({ stable, oxide }) => { config ${{ purge: [{ raw: 'text-center' }] }} ${{ purge: { content: [{ raw: 'text-center' }] } }} - ${{ content: { content: [{ raw: 'text-center' }] } }} + ${{ content: { files: [], content: [{ raw: 'text-center' }] } }} `('should normalize content $config', ({ config }) => { return run('@tailwind utilities', config).then((result) => { return expect(result.css).toMatchFormattedCss(css` @@ -22,7 +22,7 @@ crosscheck(({ stable, oxide }) => { config ${{ purge: { safelist: ['text-center'] } }} ${{ purge: { options: { safelist: ['text-center'] } } }} - ${{ content: { safelist: ['text-center'] } }} + ${{ content: { files: [], safelist: ['text-center'] } }} `('should normalize safelist $config', ({ config }) => { return run('@tailwind utilities', config).then((result) => { return expect(result.css).toMatchFormattedCss(css` diff --git a/tests/warnings.test.js b/tests/warnings.test.js index e3ea94fe552d..d2be7189f3e9 100644 --- a/tests/warnings.test.js +++ b/tests/warnings.test.js @@ -1,7 +1,9 @@ import { crosscheck, run, html, css } from './util/run' -crosscheck(() => { - test('it warns when there is no content key', async () => { +crosscheck(({ oxide, stable }) => { + // The oxide engine will use auto content if no `content` is given + oxide.test.todo('it warns when there is no content key') + stable.test('it warns when there is no content key', async () => { let config = { corePlugins: { preflight: false }, }