Redesign repo structure and get domains list from multiple sources (#44)

* refactor: design monorepos

* ci: fix ESLint chain

* ci: expand ESLint checking to all Typescript files

* refactor: split IAB sellers downloader

* refactor: update pacakge.json in `builder`

* fix: `baseUrl` in `tsconfig.json` leads fatal of `tsx` execution

* chore: add http-server utils

* chore: update .gitignore and package.json for new monorepo structure

* Potential fix for code scanning alert no. 4: Uncontrolled data used in path expression

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* chore: follow PascalCase

* Potential fix for code scanning alert no. 9: Uncontrolled data used in path expression

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

* feat: add `IndexAdShieldDomainsFromAG` func to refer AG Base filters list

* feat: add checking func to discard all CDN domains owned by Ad-Shield

* feat: add `IndexAdShieldDomainsFromUBO` func to refer uBO filters list

* feat: add `FetchAdShieldDomains` func to refer AG Base and uBO filters list

* chore: remove unnecessary import

* ci: run ESLint workflow for all branches

* chore: rename `FetchAdShieldDomainsFromFiltersLists`

* feat: add `FetchAdShieldDomains` func to refer all references

* feat: provide `CustomDefinedMatches` Set

* feat: add building cache for caching remote domains

* feat: create `CreateBanner` func

* fix: path is not resolved correctly if `npm run` is used

* feat: use new builder

* chore: add `SubscriptionUrl` customization for debugging

* fix: chokidar does not built-in  glob support

* fix: typo

---------

Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

Author: piquark6046

.github/workflows/build.yml

@@ -2,7 +2,7 @@ name: Check building and linting
 
 on:
   push:
-    branches-ignore: [ "main" ]
+    branches: [ "**" ]
   pull_request:
     # The branches below must be a subset of the branches above
     branches: [ "**" ]

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 dist
-sources/src/#generated-*.ts
+sources/src/#generated-*.ts
+.buildcache

builder.ts

@@ -0,0 +1,31 @@
+{
+  "name": "@filteringdev/tinyshield-builder",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "lint": "tsc --noEmit && eslint **/*.ts",
+    "build": "tsx source/buildci.ts",
+    "debug": "tsx source/debug.ts",
+    "clean": "rm -rf dist && rm -rf .buildcache"
+  },
+  "dependencies": {
+    "@types/node": "^24.10.9"
+  },
+  "devDependencies": {
+    "@adguard/agtree": "^3.4.3",
+    "@npmcli/package-json": "^7.0.4",
+    "@types/npmcli__package-json": "^4.0.4",
+    "@types/semver": "^7.7.1",
+    "@typescriptprime/parsing": "^1.0.4",
+    "@typescriptprime/securereq": "^1.1.0",
+    "chokidar": "^5.0.0",
+    "esbuild": "^0.27.2",
+    "eslint": "^9.39.2",
+    "semver": "^7.7.3",
+    "tldts": "^7.0.19",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "^8.53.0",
+    "zod": "^4.3.5"
+  }
+}

builder/package.json

@@ -0,0 +1,47 @@
+export interface BannerOptions {
+  Version: string
+  BuildType: 'production' | 'development'
+  Domains: Set<string>
+  Author: string
+  Name: string
+  Namespace: string
+  HomepageURL: URL
+  SupportURL: URL
+  UpdateURL: URL
+  DownloadURL: URL
+  License: string
+  Description: Record<'en' | 'ko' | 'ja' | string, string>
+}
+
+export function CreateBanner(Options: BannerOptions): string {
+  let BannerString: string = '// ==UserScript==\n'
+  BannerString += `// @name         ${Options.BuildType === 'production' ? Options.Name : Options.Name + ' (Development)'}\n`
+  BannerString += '//\n'
+  BannerString += `// @namespace    ${Options.Namespace}\n`
+  BannerString += `// @homepageURL  ${Options.HomepageURL.href}\n`
+  BannerString += `// @supportURL   ${Options.SupportURL.href}\n`
+  BannerString += `// @updateURL    ${Options.UpdateURL.href}\n`
+  BannerString += `// @downloadURL  ${Options.DownloadURL.href}\n`
+  BannerString += `// @license      ${Options.License}\n`
+  BannerString += '//\n'
+  BannerString += `// @version      ${Options.Version}\n`
+  BannerString += `// @author       ${Options.Author}\n`
+  BannerString += '//\n'
+  BannerString += '// @grant        unsafeWindow\n'
+  BannerString += '// @run-at       document-start\n'
+  BannerString += '//\n'
+  BannerString += `// @description  ${Options.Description['en']}\n`
+  
+  for (const Key of Object.keys(Options.Description)) {
+    if (Key === 'en') continue
+    BannerString += `// @description:${Key}  ${Options.Description[Key]}\n`
+  }
+  BannerString += '//\n'
+
+  for (const Domain of Options.Domains) {
+    BannerString += `// @match      *://${Domain}/*\n`
+    BannerString += `// @match      *://*.${Domain}/*\n`
+  }
+  BannerString += '// ==/UserScript==\n\n'
+  return BannerString
+}

builder/source/banner/index.ts

@@ -0,0 +1,79 @@
+import * as ESBuild from 'esbuild'
+import * as Zod from 'zod'
+import * as Process from 'node:process'
+import * as TLDTS from 'tldts'
+import PackageJson from '@npmcli/package-json'
+import { LoadDomainsFromCache } from './cache.js'
+import { FetchAdShieldDomains } from './references/index.js'
+import { CustomDefinedMatches } from './references/custom-defined.js'
+import { ConvertWildcardSuffixToRegexPattern } from './utils/wildcard-suffix-converter.js'
+import { CreateBanner } from './banner/index.js'
+
+export type BuildOptions = {
+  Minify: boolean
+  UseCache: boolean
+  BuildType: 'production' | 'development',
+  SubscriptionUrl: string,
+  Version?: string
+}
+
+export async function Build(OptionsParam?: BuildOptions): Promise<void> {
+  const Options = await Zod.strictObject({
+    Minify: Zod.boolean(),
+    UseCache: Zod.boolean(),
+    BuildType: Zod.enum(['production', 'development']),
+    SubscriptionUrl: Zod.string().transform(Value => new URL(Value)).default(new URL('https://cdn.jsdelivr.net/npm/@filteringdev/tinyshield@latest/dist/tinyShield.user.js')),
+    Version: Zod.string().optional()
+  }).parseAsync(OptionsParam)
+
+  let MatchingDomains: Set<string> = new Set<string>()
+  if (Options.UseCache) {
+    MatchingDomains = await LoadDomainsFromCache()
+  } else {
+    MatchingDomains = await FetchAdShieldDomains()
+  }
+  CustomDefinedMatches.forEach(Domain => MatchingDomains.add(Domain))
+
+  MatchingDomains = new Set<string>([...MatchingDomains].map(Domain => TLDTS.parse(Domain).domain ?? Domain).filter((D): D is string => D !== null))
+  for (const Domain of MatchingDomains) {
+    if (Domain.endsWith('.*')) {
+      MatchingDomains.delete(Domain)
+      ConvertWildcardSuffixToRegexPattern(Domain).forEach(GeneratedPattern => MatchingDomains.add(GeneratedPattern))
+    }
+  }
+
+  let ProjectRoot = Process.cwd()
+  if (Process.cwd().endsWith('/builder')) {
+    ProjectRoot = Process.cwd() + '/..'
+  }
+  
+  const Banner = CreateBanner({
+    Version: Options.Version ?? (await PackageJson.load(ProjectRoot)).content.version ?? '0.0.0',
+    BuildType: Options.BuildType ?? 'production',
+    Domains: MatchingDomains,
+    Name: 'tinyShield',
+    Namespace: 'https://github.com/FilteringDev/tinyShield',
+    DownloadURL: Options.SubscriptionUrl,
+    UpdateURL: Options.SubscriptionUrl,
+    HomepageURL: new URL('https://github.com/FilteringDev/tinyShield'),
+    SupportURL: new URL('https://github.com/FilteringDev/tinyShield/issues'),
+    License: 'MPL-2.0',
+    Author: 'PiQuark6046 and contributors',
+    Description: {
+      en: 'tinyShield allows AdGuard, uBlock Origin, Brave and ABP to resist against Ad-Shield quickly.',
+      ko: 'tinyShield는 AdGuard, uBlock Origin, Brave 와 ABP가 애드쉴드에 빠르게 저항할 수 있도록 합니다.',
+      ja: 'tinyShieldを使うと、AdGuard, uBlock Origin, Brave, およびABPがAd-Shieldに素早く対抗できます。'
+    }
+  })
+
+  await ESBuild.build({
+    entryPoints: [ProjectRoot + '/userscript/source/index.ts'],
+    bundle: true,
+    minify: Options.Minify,
+    outfile: `${ProjectRoot}/dist/tinyShield${Options.BuildType === 'development' ? '.dev' : ''}.user.js`,
+    banner: {
+      js: Banner
+    },
+    target: ['es2024', 'chrome119', 'firefox142', 'safari26']
+  })
+}

builder/source/build.ts

@@ -0,0 +1,14 @@
+import * as Zod from 'zod'
+import * as Process from 'node:process'
+import { PreProcessing, PostProcessing } from '@typescriptprime/parsing'
+import { Build, BuildOptions } from './build.js'
+
+let ParsedArgv = (await PostProcessing<BuildOptions>(PreProcessing(Process.argv))).Options
+let Options = await Zod.strictObject({
+  Minify: Zod.string().pipe(Zod.enum(['true', 'false'])).transform(Value => Value === 'true').default(true),
+  UseCache: Zod.string().pipe(Zod.enum(['true', 'false'])).transform(Value => Value === 'true').default(true),
+  BuildType: Zod.enum(['production', 'development']),
+  SubscriptionUrl: Zod.string()
+}).parseAsync(ParsedArgv)
+
+await Build(Options)

builder/source/buildci.ts

@@ -0,0 +1,46 @@
+import * as Zod from 'zod'
+import * as Fs from 'node:fs'
+import * as Process from 'node:process'
+import { FetchAdShieldDomains } from './references/index.js'
+
+const CachePath = Process.cwd() + '/.buildcache'
+const CacheDomainsPath = CachePath + '/domains.json'
+
+export function CreateCache(Domains: Set<string>) {
+  if (!Fs.existsSync(CachePath)) {
+    Fs.mkdirSync(CachePath)
+  } else if (!Fs.statSync(CachePath).isDirectory()) {
+    throw new Error('.buildcache exists and is not a directory!')
+  }
+  if (Fs.existsSync(CacheDomainsPath)) {
+    throw new Error('Cache already exists!')
+  }
+  Fs.writeFileSync(CacheDomainsPath, JSON.stringify([...Domains], null, 2), { encoding: 'utf-8' })
+}
+
+export async function LoadCache(): Promise<Set<string>> {
+  if (!Fs.existsSync(CacheDomainsPath)) {
+    throw new Error('Cache does not exist!')
+  }
+  const DomainsRaw = Fs.readFileSync(CacheDomainsPath, { encoding: 'utf-8' })
+  const DomainsArray: string[] = JSON.parse(DomainsRaw)
+  await Zod.array(Zod.string().refine((Value) => {
+    try {
+      new URLPattern(`https://${Value}/`)
+      return true
+    } catch {
+      return false
+    }
+  })).parseAsync(DomainsArray)
+  return new Set(DomainsArray)
+}
+
+export async function LoadDomainsFromCache(): Promise<Set<string>> {
+  if (!Fs.existsSync(CacheDomainsPath)) {
+    const Domains = await FetchAdShieldDomains()
+    CreateCache(Domains)
+    return Domains
+  } else {
+    return await LoadCache()
+  }
+}