From 31915e432b171e73dfeec51064d0c6c4895affff Mon Sep 17 00:00:00 2001 From: Krzysztof Moch Date: Sun, 13 Jul 2025 18:48:54 +0200 Subject: [PATCH] docs: Add `llms.txt` and `llms-full.txt` (#4603) Co-authored-by: Pieczasz --- .github/actions/setup-node/action.yml | 2 +- .github/workflows/deploy-docs.yml | 5 + .github/workflows/test-build-docs.yml | 5 + docs/.gitignore | 4 +- .../gradle/wrapper/gradle-wrapper.properties | 2 +- package.json | 2 + scripts/generate-llms-full.js | 68 +++++++ scripts/generate-llms.js | 173 ++++++++++++++++++ 8 files changed, 258 insertions(+), 3 deletions(-) create mode 100644 scripts/generate-llms-full.js create mode 100644 scripts/generate-llms.js diff --git a/.github/actions/setup-node/action.yml b/.github/actions/setup-node/action.yml index ec4ef52a..03010edc 100644 --- a/.github/actions/setup-node/action.yml +++ b/.github/actions/setup-node/action.yml @@ -13,7 +13,7 @@ runs: - name: Setup Node.js uses: actions/setup-node@v3 with: - node-version: 18.x + node-version: 20.x - name: Cache dependencies id: yarn-cache diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 2c9c3ac7..607a03be 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -31,6 +31,11 @@ jobs: ${{ runner.os }}-nextjs-${{ hashFiles('**/bun.lockb') }} ${{ runner.os }}-nextjs- + - name: Generate llms.txt and llms-full.txt + run: | + bun run docs:llms + bun run docs:llms-full + - name: Build docs run: | bun --cwd docs build diff --git a/.github/workflows/test-build-docs.yml b/.github/workflows/test-build-docs.yml index 052814c8..da5ecb08 100644 --- a/.github/workflows/test-build-docs.yml +++ b/.github/workflows/test-build-docs.yml @@ -29,6 +29,11 @@ jobs: ${{ runner.os }}-nextjs-${{ hashFiles('**/bun.lockb') }} ${{ runner.os }}-nextjs- + - name: Generate llms.txt and llms-full.txt + run: | + bun run docs:llms + bun run docs:llms-full + - name: Build docs run: | bun --cwd docs build diff --git a/docs/.gitignore b/docs/.gitignore index cf4cfe2a..21f29f1e 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,3 +1,5 @@ node_modules/ out/ -.next/ \ No newline at end of file +.next/ +public/llms.txt +public/llms-full.txt \ No newline at end of file diff --git a/examples/bare/android/gradle/wrapper/gradle-wrapper.properties b/examples/bare/android/gradle/wrapper/gradle-wrapper.properties index e2847c82..cea7a793 100644 --- a/examples/bare/android/gradle/wrapper/gradle-wrapper.properties +++ b/examples/bare/android/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/package.json b/package.json index 114060f8..26e11ab0 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,8 @@ "prepare": "yarn build", "xbasic": "yarn --cwd examples/basic", "docs": "yarn --cwd docs build", + "docs:llms": "node scripts/generate-llms.js", + "docs:llms-full": "node scripts/generate-llms-full.js", "release": "release-it", "test": "echo no test available", "check-ios": "scripts/swift-format.sh && scripts/swift-lint.sh && scripts/clang-format.sh", diff --git a/scripts/generate-llms-full.js b/scripts/generate-llms-full.js new file mode 100644 index 00000000..bfd36b8d --- /dev/null +++ b/scripts/generate-llms-full.js @@ -0,0 +1,68 @@ +#!/usr/bin/env node +import fs from 'fs'; +import path from 'path'; + +const CWD = process.cwd(); +const STATIC_DIR = path.join(CWD, 'docs', 'public'); + +const LLMS_FILE = path.join(STATIC_DIR, 'llms.txt'); +const OUTPUT_PATH_STATIC = path.join(STATIC_DIR, 'llms-full.txt'); + +function readIfExists(p) { + return fs.existsSync(p) ? fs.readFileSync(p, 'utf8') : ''; +} + +function* walk(dir) { + for (const entry of fs.readdirSync(dir, {withFileTypes: true})) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + yield* walk(full); + } else if (/\.(md|mdx)$/.test(entry.name)) { + yield full; + } + } +} + +function gatherDocs() { + // Crawl all markdown files in the docs directory (excluding the static subdir) + const docsDir = path.join(CWD, 'docs', 'pages'); + if (!fs.existsSync(docsDir)) { + throw new Error(`Docs directory not found! Tried to read ${docsDir}`); + } + + return Array.from(walk(docsDir)).sort(); +} + +function renderFileContent(filepath) { + const rel = path.relative(CWD, filepath); + const content = readIfExists(filepath); + return `\n\n## ${rel}\n\n${content}\n`; +} + +function main() { + const parts = []; + // 1. quick overview + parts.push(readIfExists(LLMS_FILE)); + + // 2. root README & package README (if present) + const rootReadme = path.join(CWD, 'README.md'); + if (fs.existsSync(rootReadme)) { + parts.push('\n\n## README (root)\n\n' + readIfExists(rootReadme)); + } + + // 3. all docs + for (const file of gatherDocs()) { + parts.push(renderFileContent(file)); + } + + const output = parts.join('\n'); + + fs.mkdirSync(STATIC_DIR, {recursive: true}); + // Write to docs/public so it will be available at /llms-full.txt + fs.writeFileSync(OUTPUT_PATH_STATIC, output, 'utf8'); + console.log( + `✔︎ wrote llms-full.txt (size: ${output.length.toLocaleString()} chars)`, + ); +} + +main(); diff --git a/scripts/generate-llms.js b/scripts/generate-llms.js new file mode 100644 index 00000000..3707cd3c --- /dev/null +++ b/scripts/generate-llms.js @@ -0,0 +1,173 @@ +#!/usr/bin/env node +import fs from 'fs'; +import path from 'path'; + +/** Convenience utils */ +const CWD = process.cwd(); + +/** Simple helper that returns the first N lines of README.md (sans markdown headings). */ +function getProjectOverview(maxLines = 30) { + const readmePath = path.join(CWD, 'README.md'); + if (!fs.existsSync(readmePath)) { + return ''; + } + const lines = fs.readFileSync(readmePath, 'utf8').split(/\r?\n/); + return lines.slice(0, maxLines).join('\n'); +} + +/** Recursively build a filtered tree representation – depth-limited for brevity. */ +function buildFileTree(startPath, depth = 0, maxDepth = 2) { + const ignore = new Set([ + 'node_modules', + '.git', + 'build', + 'android', + 'ios', + '.docusaurus', + '.next', + '.expo', + ]); + + if (depth > maxDepth) { + return []; + } + const entries = fs.readdirSync(startPath, {withFileTypes: true}); + const lines = []; + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + if (ignore.has(entry.name) || entry.name.startsWith('.')) { + continue; + } + const relPath = `${' '.repeat(depth)}- ${entry.name}${ + entry.isDirectory() ? '/' : '' + }`; + lines.push(relPath); + if (entry.isDirectory()) { + const childLines = buildFileTree( + path.join(startPath, entry.name), + depth + 1, + maxDepth, + ); + lines.push(...childLines); + } + } + return lines; +} + +/** Minimal usage examples that are helpful for agents. */ +function getUsageExamples() { + return "1. Basic playback\n\n ```tsx\n import Video, { VideoRef } from 'react-native-video';\n import { useRef } from 'react';\n \n export default function Example() {\n const videoRef = useRef(null);\n \n return (\n \n );\n }\n ```\n\n2. Advanced control (seek & pause)\n\n ```tsx\n // Using ref to control playback\n videoRef.current?.setNativeProps({ paused: true });\n videoRef.current?.seek(10); // seconds\n ```"; +} + +/** Load docs metadata and build navigation structure */ +function generateDocsSections() { + const docsRoot = path.join(CWD, 'docs', 'pages'); + if (!fs.existsSync(docsRoot)) { + return []; + } + const baseUrl = 'https://docs.thewidlarzgroup.com/react-native-video'; + + const sections = []; + const dirEntries = fs.readdirSync(docsRoot, {withFileTypes: true}); + for (const entry of dirEntries) { + const absPath = path.join(docsRoot, entry.name); + if (entry.isDirectory()) { + const title = toTitle(entry.name); + const items = collectMarkdownLinks(absPath, `${baseUrl}/${entry.name}`); + if (items.length) { + sections.push({title, items}); + } + } else if (entry.isFile() && entry.name.endsWith('.md')) { + // root-level docs + const title = + getMarkdownTitle(absPath) || toTitle(entry.name.replace(/\.mdx?$/, '')); + sections.push({ + title: 'General', + items: [ + {title, url: `${baseUrl}/${entry.name.replace(/\.mdx?$/, '')}`}, + ], + }); + } + } + return sections; +} + +function toTitle(slug) { + return slug + .replace(/[-_]/g, ' ') + .replace(/\b\w/g, (c) => c.toUpperCase()) + .trim(); +} + +function getMarkdownTitle(filePath) { + const content = fs.readFileSync(filePath, 'utf8'); + const match = content.match(/^#\s+(.+)/m); + return match ? match[1].trim() : null; +} + +function collectMarkdownLinks(dir, urlPrefix) { + const links = []; + for (const file of fs.readdirSync(dir)) { + if (!file.endsWith('.md') && !file.endsWith('.mdx')) { + continue; + } + if (file.startsWith('_')) { + continue; // skip _category_.json etc + } + const abs = path.join(dir, file); + const title = getMarkdownTitle(abs) || toTitle(file.replace(/\.mdx?$/, '')); + links.push({ + title, + url: `${urlPrefix}/${file.replace(/\.mdx?$/, '')}`, + }); + } + return links; +} + +function formatDocsSections(sections) { + if (!sections.length) { + return ''; + } + let md = '\n## Documentation\n'; + sections.forEach((sec) => { + md += `\n### ${sec.title}\n`; + sec.items.forEach((it) => { + md += `- [${it.title}](${it.url})\n`; + }); + }); + return md + '\n'; +} + +function generateContent() { + const overview = getProjectOverview(); + const treeLines = buildFileTree(path.join(CWD), 0, 2).join('\n'); + const usage = getUsageExamples(); + const docsSections = formatDocsSections(generateDocsSections()); + + return `# react-native-video\n\n> A