Add action (#3)

Co-authored-by: Michael MIGLIORE <[email protected]>

.github/workflows/check_size.cmake

@@ -0,0 +1,5 @@
+# Check that a LFS data file is big enough to be an actual file
+file(SIZE "${CMAKE_ARGV3}" lfs_file_size)
+if (lfs_file_size LESS_EQUAL 500)
+  message(FATAL_ERROR "File is smaller than 500 bytes, lfs data not correctly recovered")
+endif()

.github/workflows/ci.yml

@@ -0,0 +1,75 @@
+name: CI
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+  push:
+    branches:
+      - main
+
+concurrency:
+  group: '${{ github.workflow }}-${{ github.ref_name }}'
+  cancel-in-progress: true
+
+jobs:
+  cache_lfs:
+    runs-on: ubuntu-latest
+    name: Update LFS data cache
+    outputs:
+      lfs_sha: ${{ steps.lfs_sha.outputs.lfs_sha }}
+    steps:
+
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        path: 'lfs-data-cache_action'
+        fetch-depth: 0
+
+    - name: Use lfs-data-cache action
+      id: lfs-data-cache
+      uses: ./lfs-data-cache_action
+      with:
+        type: 'producer' 
+        repository: 'f3d-app/f3d'
+        cache_postfix: 'ci-cache'
+
+    - name: Set output
+      id: lfs_sha
+      shell: bash
+      run: echo "lfs_sha=$(steps.lfs-data-cache.outputs.lfs_sha)" >> $GITHUB_OUTPUT
+
+  ci:
+    name: CI
+    needs: cache_lfs
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-13]
+
+    runs-on: ${{matrix.os}}
+
+    steps:
+
+    - name: Output directory
+      shell: bash
+      run: mkdir output_dir
+
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        path: 'lfs-data-cache_action'
+        fetch-depth: 0
+
+    - name: Use lfs-data-cache action
+      uses: ./lfs-data-cache_action
+      with:
+        type: 'consumer' 
+        repository: 'f3d-app/f3d'
+        lfs_sha: ${{ needs.cache_lfs.outputs.lfs_sha}}
+        cache_postfix: 'ci-cache'
+        target_directory: 'output_dir'
+
+    - name: Check output has expected size
+      shell: bash
+      run: cmake -P ./lfs-data-cache_action/.github/workflows/check_size.cmake output_dir/testing/data/f3d.vtp

README.md

@@ -1,2 +1,91 @@
 # lfs-data-cache-action
-A github action to cache and recover LFS data
+
+A producer/consumer github action to cache and recover LFS data.
+
+Its main use is to avoid cloning LFS data in CI to avoid
+having to pay for LFS bandwidth because of CI needs.
+
+It expects cmake to be available on the host.
+
+The action can be used as a consumer or a producer, and must
+provide the repository containing the LFS data to recover from.
+
+It is possible to provide a specific SHA to produce.
+If not provided, the last commit modyfing the LFS file will be produced.
+
+Is has the following inputs:
+
+ - `type`: should be `producer` or `consumer`, default to `producer`
+ - `repository`: the git repository to produce LFS data from, default: ${{ github.repository }}
+ - `lfs_sha`: The git sha to recover LFS data from, optional
+ - `cache_postfix`: An postfix added to the cache name, to support multiple caches, default to `cache`
+ - `target_directory`: A target directory to copy LFS data to
+
+## Logic
+
+Producer/Consumer first use the classic cache action to recover a cache named
+`lfs-data-${{lfs_sha}}-${{cache_index}}`.
+
+If Producer does not found it, it will clone the `repository` at `lfs_sha` commit
+and upload the content as an artifact.
+
+If Consumer does not found it, it will try to download a potential artifact
+produced earlier by the Producer.
+
+If it fails Consumer will clone the `repository` at `lfs_sha` commit.
+
+Finally, Producer/Consumer will copy the LFS data only using cmake to the `target_directory`
+
+## Usage
+
+In an first job, use the `producer` action, which output the LFS sha that will be produced
+In a second job, usually a matrix job, depending on the first,
+recover the LFS sha from first job and use the `consumer` action.
+
+```
+jobs:
+
+#----------------------------------------------------------------------------
+# Cache LFS: Checkout LFS data and update the cache to limit LFS bandwidth
+#----------------------------------------------------------------------------
+  cache_lfs:
+    runs-on: ubuntu-latest
+    name: Update LFS data cache
+    outputs:
+      lfs_sha: ${{ steps.lfs_sha_recover.outputs.lfs_sha }}
+    steps:
+
+    # Checkout your repository WITHOUT LFS
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        path: 'source'
+        fetch-depth: 1
+        lfs: false
+
+    # Use producer action to recover the LFS data and upload it as cache/artifacts
+    - name: Cache LFS Data
+      id: lfs_sha_recover
+      uses: f3d-app/lfs-data-cache-action:v1
+
+#----------------------------------------------------------------------------
+# Actual CI: Recover LFS data first
+#----------------------------------------------------------------------------
+
+  recover_lfs:
+    needs: cache_lfs
+
+    # Checkout your repository WITHOUT LFS
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        path: 'source'
+        fetch-depth: 0
+        lfs: false
+
+    - name: Recover LFS Data
+      uses: f3d-app/lfs-data-cache-action:v1
+      with:
+        workflow_label: 'consumer'
+        lfs_sha: ${{ needs.cache_lfs.outputs.lfs_sha}}
+```

action.yml

@@ -0,0 +1,121 @@
+name: 'Copy LFS data'
+description: 'Copy LFS data using cache when possible'
+inputs:
+  type:
+    description: 'Whether this action produce/consume LFS data'
+    default: 'producer'
+  repository:
+    description: 'Repository to recover LFS data from'
+    required: false
+    default: ${{ github.repository }}
+  lfs_sha:
+    description: 'LFS sha to recover. If not set, target directory will be used to recover it.'
+    required: false
+  cache_postfix:
+    description: 'A postfix to differentiate between caches'
+    default: 'cache'
+  target_directory:
+    description: 'Existing directory to copy LFS data to'
+    default: 'source'
+outputs:
+  lfs_sha:
+    description: 'The lfs_sha generated by a producer action'
+    value: ${{ steps.lfs_sha_recover.outputs.lfs_sha }}
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Check required inputs
+      shell: bash
+      run: |
+        [[ "${{ inputs.repository }}" ]] || { echo "repository input is empty" ; exit 1; }
+
+    - name: Create a working directory
+      working-directory: ${{github.workspace}}
+      shell: bash
+      run: mkdir lfs_data_cache
+
+    - name: Checkout repository without LFS data
+      if: inputs.lfs_sha == ''
+      uses: actions/checkout@v4
+      with:
+        repository: ${{ inputs.repository }}
+        path: 'lfs_data_cache/lfs_source'
+        fetch-depth: 0
+        lfs: false
+
+    - name: Set LFS sha env var from repository
+      working-directory: ${{github.workspace}}/lfs_data_cache/lfs_source
+      if: inputs.lfs_sha == ''
+      shell: bash
+      run: echo "lfs_data_cache_sha=$(git log -n 1 --pretty=format:%H -- `git-lfs ls-files -n`)" >> $GITHUB_ENV
+
+    - name: Set LFS sha env var from inputs
+      if: inputs.lfs_sha != ''
+      shell: bash
+      run: echo "lfs_data_cache_sha=${{inputs.lfs_sha}}" >> $GITHUB_ENV
+
+    - name: Cache LFS data
+      id: cache-lfs
+      uses: actions/cache@v4
+      with:
+        path: 'lfs_data_cache/lfs_data'
+        key: lfs-data-${{env.lfs_data_cache_sha}}-${{inputs.cache_postfix}}
+
+    - name: Set LFS output sha
+      id: lfs_sha_recover
+      shell: bash
+      run: echo "lfs_sha=${{env.lfs_data_cache_sha}}" >> $GITHUB_OUTPUT
+
+    - name: Checkout LFS data for artifact producer
+      if: |
+        steps.cache-lfs.outputs.cache-hit != 'true' &&
+        inputs.type == 'producer'
+      uses: actions/checkout@v4
+      with:
+        repository: ${{ inputs.repository }}
+        ref: ${{env.lfs_data_cache_sha}}
+        path: 'lfs_data_cache/lfs_data'
+        fetch-depth: 0
+        lfs: true
+
+    - name: Upload LFS artifact
+      if: |
+        steps.cache-lfs.outputs.cache-hit != 'true' &&
+        inputs.type == 'producer'
+      uses: actions/upload-artifact@v4
+      with:
+        name: lfs-data-${{inputs.cache_postfix}}
+        path: 'lfs_data_cache/lfs_data'
+        overwrite: true
+        include-hidden-files: true
+
+    - name: Download LFS artifact
+      id: download-artifact
+      if: |
+        steps.cache-lfs.outputs.cache-hit != 'true' &&
+        inputs.type == 'consumer'
+      uses: actions/download-artifact@v4
+      continue-on-error: true
+      with:
+        name: lfs-data-${{inputs.cache_postfix}}
+        path: 'lfs_data_cache/lfs_data'
+
+    - name: Checkout LFS data (last resort)
+      if: |
+        steps.cache-lfs.outputs.cache-hit != 'true' &&
+        steps.download-artifact.outcome != 'success' &&
+        inputs.type == 'consumer'
+      uses: actions/checkout@v4
+      with:
+        repository: ${{ inputs.repository }}
+        ref: ${{env.lfs_data_cache_sha}}
+        path: 'lfs_data_cache/lfs_data'
+        fetch-depth: 0
+        lfs: true
+
+    - name: Setup LFS data
+      working-directory: ${{github.workspace}}
+      shell: bash
+      run: cmake -P $GITHUB_ACTION_PATH/copy_lfs.cmake 'lfs_data_cache/lfs_data' ${{ inputs.target_directory }}

copy_lfs.cmake

@@ -0,0 +1,12 @@
+find_package(Git QUIET)
+execute_process(
+    COMMAND ${GIT_EXECUTABLE} lfs ls-files -n 
+    WORKING_DIRECTORY ${CMAKE_ARGV3}
+    OUTPUT_VARIABLE LFS_FILES
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+string(REPLACE "\n" ";" LFS_FILES ${LFS_FILES})
+
+foreach(LFS_FILE ${LFS_FILES})
+  cmake_path(GET LFS_FILE PARENT_PATH LFS_PATH)
+  file(COPY "${CMAKE_ARGV3}/${LFS_FILE}" DESTINATION "${CMAKE_ARGV4}/${LFS_PATH}")
+endforeach()