Disha/GitHub_ci_nf_test

.github/workflows/linting.yaml

@@ -0,0 +1,105 @@
+name: Run Linting
+on: [push, pull_request]
+
+env:
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
+  NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  NXF_VER: "25.10.2"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  nf-core-changes:
+    name: Detect changes
+    runs-on: ubuntu-latest
+
+    outputs:
+      # https://github.com/dorny/paths-filter?tab=readme-ov-file#custom-processing-of-changed-files
+      modules: ${{ steps.filter.outputs.modules }}
+      modules_files: ${{ steps.module_names.outputs.result }}
+      subworkflows: ${{ steps.filter.outputs.subworkflows }}
+      subworkflows_files: ${{ steps.subworkflow_names.outputs.result }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4
+        id: filter
+        with:
+          filters: |
+            modules:
+              - added|modified: 'modules/ensembl/**'
+            subworkflows:
+              - added|modified: 'subworkflows/ensembl/**'
+          token: ""
+          list-files: "json"
+
+      - name: Get module name
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        id: module_names
+        with:
+          script: |
+            return [...new Set(${{ steps.filter.outputs.modules_files }}
+              .filter(x => x.endsWith('main.nf') || x.endsWith('.nf.test.snap'))
+              .map(path => path
+                .replace('modules/ensembl/', '')
+                .replace(/\/(main\.nf|tests\/.*)$/, '')
+              )
+            )];
+      - name: Get subworkflow name
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
+        id: subworkflow_names
+        with:
+          script: |
+            return [...new Set(${{ steps.filter.outputs.subworkflows_files }}
+              .filter(x => x.endsWith('main.nf') || x.endsWith('.nf.test.snap'))
+              .map(path => path
+                .replace('subworkflows/nf-core/', '')
+                .replace(/\/(main\.nf|tests\/.*)$/, '')
+              )
+            )];
+
+      - name: debug
+        run: |
+          echo ${{ steps.filter.outputs.modules_files }}
+          echo ${{ steps.module_names.outputs.result }}
+          echo ${{ steps.filter.outputs.subworkflows_files }}
+          echo ${{ steps.subworkflow_names.outputs.result }}
+
+
+  nf-core-lint:
+    name: nf-core lint
+    runs-on: ubuntu-latest
+    needs: nf-core-changes
+    if: needs.nf-core-changes.outputs.modules_files != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        module: ${{ fromJson(needs.nf-core-changes.outputs.modules_files) }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+
+      - uses: actions/cache@v5
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip
+
+      - uses: nf-core/setup-nextflow@v2
+
+      - name: Install nf-core tools
+        run: pip install --upgrade git+https://github.com/nf-core/tools.git@dev
+
+      - name: Lint modules
+        run: nf-core modules lint ${{ matrix.module }}
+
+      - name: Lint subworkflows
+        if: needs.nf-core-changes.outputs.subworkflows == 'true'
+        run: nf-core subworkflows lint --all

.github/workflows/nf-test.yaml

@@ -0,0 +1,48 @@
+name: NF Tests
+on: [push, pull_request]
+
+env:
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
+  NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
+  NFTEST_VER: "0.9.0"
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: '11'
+          distribution: 'adopt'
+
+      - name: Setup Nextflow latest
+        uses: nf-core/setup-nextflow@v2
+        with:
+          version: "latest-stable"
+
+      - name: Install nf-test
+        uses: nf-core/setup-nf-test@v1
+        with:
+          version: ${{ env.NFTEST_VER }}
+
+      - name: Setup apptainer
+        uses: eWaterCycle/setup-apptainer@main
+
+      - name: Set up Singularity
+        run: |
+          mkdir -p $NXF_SINGULARITY_CACHEDIR
+          mkdir -p $NXF_SINGULARITY_LIBRARYDIR
+
+      - name: Set up Python
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
+        with:
+          python-version: "3.11"
+
+      - name: Run Tests
+        run: nf-test test --ci #this is used to prevent updating the snapshot 

modules/ensembl/annotation/dumpanno/main.nf

@@ -50,7 +50,7 @@ process ANNOTATION_DUMPANNO {
     stub:
         version = "0.4"
         """
-        echo "No change, create an empty json file" > functional_annotation.json
+        touch functional_annotation.json
 
         # Get version from genomio please
         echo -e -n "${task.process}:\n\tensembl-legacy-scripts:e112_APIv0.4 : $version" > versions.yml

modules/ensembl/annotation/dumpanno/meta.yml

@@ -1,51 +1,47 @@
 ---
-name: "annotation_dumpanno"
-
-description: This module is used to dump the functional annotation of a genome from core databases.
+name: "database_dbfactory"
+description: >
+  Generate a list of one or more databases plus some associated meta table information.
 
 keywords:
   - container
   - core_database
   - docker
-  - ensembl-legacy-scripts
-  - fasta
-  
+  - database
+  - ensembl-genomio
+
 tools:
-  - "annotation":
-      homepage: "https://github.com/Ensembl/nextflow_modules"
-      description: "This module handles all annotation-related interactions with the core database, including annotation dumping and processing"
-      license: ['Apache License version 2.0']
+  database:
+    description: >
+      Modules which function to read or write information from core databases.
+    homepage: "https://github.com/Ensembl/nextflow_modules"
+    license: ["Apache License version 2.0"]
+    identifier: ""
 
 input:
-  # Only when we have meta
-  - meta:
-        type: map
-        description: |
-          A meta map including 'core database information'.
-          e.g. [ id:'db_name', server_details:[ host:'db' ] ]
-        pattern: "[a-z]+_[a-z]_gc[af][0-9]v[0-9]+_core_*"
+  - name: server
+    type: map
+    description: >
+      Groovy Map containing MySQL server host, port, user, and password meta information,
+      e.g. `[ host:"mysql-host-test-prod" ]`
 
-output:
-  - meta:
-      type: map
-      description: |
-        Database containing meta information.
-        e.g. [ id:'db_name', server_details:[ host:'db' ] ]
-      pattern: "[a-z]+_[a-z]_gc[af][0-9]v[0-9]+_core_*"
+  - name: filter_map
+    type: file
+    description: "JSON file containing database filter information."
+    pattern: "*.json"
 
-  - functional_annotation.json:
-      type: map
-      description: |
-        Functional annotation file containing biotypes.
-        e.g [ db:'database name', file("*.json") ]
-      pattern: ".json"
+output:
+  - name: dbs_meta
+    type: file
+    description: "JSON file containing database metadata."
+    pattern: "dbs.json"
 
-  - versions:
-      type: file
-      description: File containing software versions.
-      pattern: "versions.yml"
+  - name: versions
+    type: file
+    description: "File containing information on module software version"
+    pattern: "versions.yml"
 
 authors:
-  - "@ensembl-dev"
+  - "ensembl-dev@ebi.ac.uk"
 maintainers:
-  - "@ensembl-dev"
+  - "ensembl-dev@ebi.ac.uk"

modules/ensembl/database/dbfactory/meta.yml

@@ -1,6 +1,7 @@
 ---
 name: "database_dbfactory"
-description: Generate a list of one or more databases plus some associated meta table information.
+description: >
+  Generate a list of one or more databases plus some associated meta table information.
 
 keywords:
   - container
@@ -10,30 +11,35 @@ keywords:
   - ensembl-genomio
 
 tools:
-  - database:
-      description: "Modules which function to read or write information from core databases."
-      homepage: "https://github.com/Ensembl/nextflow_modules"
-      license: ['Apache License version 2.0']
+  database:
+    description: >
+      Modules which function to read or write information from core databases.
+    homepage: "https://github.com/Ensembl/nextflow_modules"
+    license: ["Apache License version 2.0"]
+    identifier: ""
 
 input:
-  - server:
-      type: map
-      description: "Groovy Map containing mysql server host, port, user and password 
-        meta and file of JSON database filter information."
-        e.g. [[ host:"mysql-host-test-prod" ], [ filter_map:"dbrename_re" ]]
-  - filtermap:
-      type: map
-      description: "Prefix to filter core databases. [default: none]"
+  - name: server
+    type: map
+    description: >
+      Groovy Map containing MySQL server host, port, user, and password meta information,
+      e.g. `[ host:"mysql-host-test-prod" ]`
+
+  - name: filter_map
+    type: file
+    description: "JSON file containing database filter information."
+    pattern: "*.json"
 
 output:
-  - dbs_meta_json:
-      type: file
-      description: "JSON file containing database metadata."
-      pattern: "dbs.json"  
-  - versions:
-      type: file
-      description: File containing information on module software version
-      pattern: "versions.yml"
+  - name: dbs_meta
+    type: file
+    description: "JSON file containing database metadata."
+    pattern: "dbs.json"
+
+  - name: versions
+    type: file
+    description: "File containing information on module software version"
+    pattern: "versions.yml"
 
 authors:
   - "ensembl-dev@ebi.ac.uk"

modules/ensembl/download/assemblydata/meta.yml

@@ -1,6 +1,7 @@
 ---
 name: "download_assemblydata"
-description: Generate a list of one or more databases plus some associated meta table information.
+description: >
+  Generate a list of one or more databases plus some associated meta table information.
 
 keywords:
   - ensembl-genomio
@@ -12,35 +13,38 @@ keywords:
   - INSDC
 
 tools:
-  - download:
-      description: "Modules which function to download information or various data from externally hosted resources."
-      homepage: "https://github.com/Ensembl/nextflow_modules"
-      license: ['Apache License version 2.0']
+  download:
+    description: >
+      Modules which function to download information or various data from
+      externally hosted resources.
+    homepage: "https://github.com/Ensembl/nextflow_modules"
+    license: ["Apache License version 2.0"]
+    identifier: ""
 
 input:
-  - meta:
-      type: map
-      description: |
-        Groovy Map containing A single INSDC GCA or NCBI 'amended' GCF genome assembly accession meta information.
-        e.g. `[ accession:'GCA_000000000.1' ]`
+  - name: meta
+    type: map
+    description: >
+      Groovy Map containing a single INSDC GCA or NCBI 'amended' GCF genome assembly accession meta information,
+      e.g. `[ accession:'GCA_000000000.1' ]`
 
 output:
-  - min_set:
-      type: map
-      description: "Groovy Map including various genome assembly data files (.txt, .fna, .gbff) into compressed Gzip archives."
-        e.g. [ accession:'GCA_000000000.1', file("*_assembly_report.txt"), file("*_genomic.fna.gz"), file("*_genomic.gbff.gz") ]
-      pattern: (*.txt, *.fna.gz, *.gbff.gz)
-  - opt_set:
-      type: map
-      description: "Groovy Map including various genome assembly data files (.txt, .faa, .gff) into compressed Gzip archives."
-        e.g. [ accession:'GCA_000000000.1', file("*_assembly_report.txt"), file("*_genomic.gff.gz"), file("*_genomic.gbff.gz") ]
-      pattern: (*.txt, *.fna.gz, *.gbff.gz)
-  - versions:
-      type: file
-      description: File containing information on module software version
-      pattern: "versions.yml"
+  - name: min_set
+    type: file
+    description: "Minimal set of downloaded assembly data files"
+    pattern: "*"
+
+  - name: opt_set
+    type: file
+    description: "Optional set of downloaded assembly data files"
+    pattern: "*"
+
+  - name: versions
+    type: file
+    description: "File containing information on module software version"
+    pattern: "versions.yml"
 
 authors:
   - "ensembl-dev@ebi.ac.uk"
 maintainers:
-  - "ensembl-dev@ebi.ac.uk"
+  - "ensembl-dev@ebi.ac.uk"