colmap · ahojnnes · Jun 22, 2023 · Jun 22, 2023 · Jun 22, 2023 · Jun 22, 2023
diff --git a/.azure-pipelines/build-mac.yaml b/.azure-pipelines/build-mac.yaml
@@ -1,12 +1,15 @@
 parameters:
-  displayName: 'Mac 10.15'
+  displayName: Mac 10.15
   macVersion: '10.15'
 
 jobs:
 - job: mac_build_${{ replace(parameters.macVersion, '.', '') }}
-  displayName: '${{ parameters.displayName }}'
+  displayName: ${{ parameters.displayName }}
   pool:
     vmImage: 'macOS-${{ parameters.macVersion }}'
+  variables:
+    CCACHE_DIR: $(Pipeline.Workspace)/ccache
+    CCACHE_BASEDIR: $(Build.SourcesDirectory)
   steps:
   - script: |
       brew install \
@@ -23,8 +26,16 @@ jobs:
         qt5 \
         glew \
         cgal \
-        sqlite3
-    displayName: 'Install dependencies'
+        sqlite3 \
+        ccache
+    displayName: Install dependencies
+
+  - task: Cache@2
+    inputs:
+      key: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
+      restoreKeys: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
+      path: $(CCACHE_DIR)
+    displayName: Restore ccache
 
   - script: |
       export PATH="/usr/local/opt/qt@5/bin:$PATH"
@@ -36,7 +47,7 @@ jobs:
         -DTESTS_ENABLED=ON \
         -DQt5_DIR=/usr/local/opt/qt/lib/cmake/Qt5
       ninja
-    displayName: 'Configure and build'
+    displayName: Configure and build
 
   - script: |
       cd build
@@ -47,4 +58,11 @@ jobs:
           ctest --rerun-failed --output-on-failure
       fi
       exit $tests_pass
-    displayName: 'Run tests'
+    displayName: Run tests
+
+  - script: |
+      set -x
+      ccache --show-stats
+      ccache --evict-older-than 1d
+      ccache --show-stats
+    displayName: Cleanup ccache
diff --git a/.azure-pipelines/build-ubuntu.yaml b/.azure-pipelines/build-ubuntu.yaml
@@ -1,16 +1,19 @@
 parameters:
-  displayName: 'Ubuntu 20.04'
+  displayName: Ubuntu 20.04
   ubuntuVersion: '20.04'
   cudaEnabled: false
   asanEnabled: false
   cmakeBuildType: Release
 
 jobs:
 - job: ubuntu_build_${{ replace(parameters.ubuntuVersion, '.', '') }}_cuda_${{ parameters.cudaEnabled }}_asan_${{ parameters.asanEnabled }}_${{ parameters.cmakeBuildType }}
-  displayName: '${{ parameters.displayName }}'
+  displayName: ${{ parameters.displayName }}
   pool:
     vmImage: 'ubuntu-${{ parameters.ubuntuVersion }}'
   variables:
+    CCACHE_DIR: $(Pipeline.Workspace)/ccache
+    CCACHE_BASEDIR: $(Build.SourcesDirectory)
+    CCACHE_VERSION: 1
     ${{ if eq(parameters.asanEnabled, false) }}:
       ctestExclusions: "(mvs/gpu_mat_test)"
     ${{ if eq(parameters.asanEnabled, true) }}:
@@ -41,7 +44,7 @@ jobs:
         libgl1-mesa-dri \
         libunwind-dev \
         xvfb
-    displayName: 'Install dependencies'
+    displayName: Install dependencies
 
   - ${{ if eq(parameters.cudaEnabled, true) }}:
     - ${{ if eq(parameters.ubuntuVersion, '22.04') }}:
@@ -53,29 +56,47 @@ jobs:
           echo '##vso[task.setvariable variable=CC]/usr/bin/gcc-10'
           echo '##vso[task.setvariable variable=CXX]/usr/bin/g++-10'
           echo '##vso[task.setvariable variable=CUDAHOSTCXX]/usr/bin/g++-10'
-        displayName: 'Install CUDA'
+        displayName: Install CUDA
     - ${{ if ne(parameters.ubuntuVersion, '22.04') }}:
       - script: |
           sudo apt-get install -y \
             nvidia-cuda-toolkit \
             nvidia-cuda-toolkit-gcc
           echo '##vso[task.setvariable variable=CC]/usr/bin/cuda-gcc'
           echo '##vso[task.setvariable variable=CXX]/usr/bin/cuda-g++'
-        displayName: 'Install CUDA'
+        displayName: Install CUDA
 
   - ${{ if eq(parameters.asanEnabled, true) }}:
     - script: |
         sudo apt-get install -y clang-15
         echo '##vso[task.setvariable variable=CC]/usr/bin/clang-15'
         echo '##vso[task.setvariable variable=CXX]/usr/bin/clang++-15'
-      displayName: 'Install Clang'
+      displayName: Install Clang
 
  - ${{ if eq(parameters.cmakeBuildType, 'ClangTidy') }}:
     - script: |
         sudo apt-get install -y clang-15 clang-tidy-15
         echo '##vso[task.setvariable variable=CC]/usr/bin/clang-15'
         echo '##vso[task.setvariable variable=CXX]/usr/bin/clang++-15'
-      displayName: 'Install Clang'
+      displayName: Install Clang
+
+  - task: Cache@2
+    inputs:
+      key: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
+      restoreKeys: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
+      path: $(CCACHE_DIR)
+    displayName: Restore ccache
+
+  - script: |
+      set -x
+      wget https://github.com/ccache/ccache/releases/download/v4.8.2/ccache-4.8.2-linux-x86_64.tar.xz
+      echo "0b33f39766fe9db67f40418aed6a5b3d7b2f4f7fab025a8213264b77a2d0e1b1  ccache-4.8.2-linux-x86_64.tar.xz" | sha256sum --check
+      tar xfv ccache-4.8.2-linux-x86_64.tar.xz
+      mkdir -p "$(CCACHE_DIR)/bin"
+      mv ./ccache-4.8.2-linux-x86_64/ccache "$(CCACHE_DIR)/bin"
+      echo "##vso[task.prependpath]$(CCACHE_DIR)/bin"
+    displayName: Install CCache
+    condition: ne(variables.CCACHE_RESTORED, 'true')
 
   - script: |
       cmake --version
@@ -88,19 +109,27 @@ jobs:
         -DCMAKE_CUDA_ARCHITECTURES=50 \
         -DASAN_ENABLED=${{ parameters.asanEnabled }}
       ninja
-    displayName: 'Configure and build'
+    displayName: Configure and build
+
+  - ${{ if ne(parameters.cmakeBuildType, 'ClangTidy') }}:
+    - script: |
+        export DISPLAY=":99.0"
+        export QT_QPA_PLATFORM="offscreen"
+        Xvfb :99 &
+        sleep 3
+        cd build
+        ctest -E "$(ctestExclusions)"
+        tests_pass=$?
+        if [ $tests_pass -ne 0 ]; then
+            echo "Tests failed, rerunning with verbose output"
+            ctest --rerun-failed --output-on-failure
+        fi
+        exit $tests_pass
+      displayName: Run tests
 
   - script: |
-      export DISPLAY=":99.0"
-      export QT_QPA_PLATFORM="offscreen"
-      Xvfb :99 &
-      sleep 3
-      cd build
-      ctest -E "$(ctestExclusions)"
-      tests_pass=$?
-      if [ $tests_pass -ne 0 ]; then
-          echo "Tests failed, rerunning with verbose output"
-          ctest --rerun-failed --output-on-failure
-      fi
-      exit $tests_pass
-    displayName: 'Run tests'
+      set -x
+      ccache --show-stats
+      ccache --evict-older-than 1d
+      ccache --show-stats
+    displayName: Cleanup ccache
diff --git a/.azure-pipelines/build-windows.yaml b/.azure-pipelines/build-windows.yaml
@@ -8,8 +8,10 @@ jobs:
   pool:
     vmImage: 'windows-${{ parameters.visualStudioVersion }}'
   variables:
-    vcpkgGitCommitId: 662dbb50e63af15baa2909b7eac5b1b87e86a0aa
+    vcpkgGitCommitId: bf64d76062b3004544731fe9c1dc3a13d5b7b59a
     VCPKG_BINARY_SOURCES: 'clear;nuget,https://pkgs.dev.azure.com/colmap/colmap/_packaging/vcpkg/nuget/v3/index.json,readwrite'
+    CCACHE_DIR: $(Pipeline.Workspace)/ccache
+    CCACHE_BASEDIR: $(Build.SourcesDirectory)
   steps:
   - task: NuGetAuthenticate@0
     displayName: NuGet Authenticate
@@ -24,31 +26,45 @@ jobs:
   - pwsh: |
       git clone https://github.com/microsoft/vcpkg
       cd vcpkg
-      git checkout --force $(vcpkgGitCommitId)
-      .\bootstrap-vcpkg.bat
-      $(Build.SourcesDirectory)\scripts\shell\enter_vs_dev_shell.ps1
-      .\vcpkg.exe install --recurse @$(Build.SourcesDirectory)/.azure-pipelines/build-windows-vcpkg.txt --clean-after-build
+      git reset --hard $(vcpkgGitCommitId)
+      ./bootstrap-vcpkg.bat
+      $(Build.SourcesDirectory)/scripts/shell/enter_vs_dev_shell.ps1
+      ./vcpkg.exe install --recurse @$(Build.SourcesDirectory)/.azure-pipelines/build-windows-vcpkg.txt --clean-after-build
     workingDirectory: $(Build.BinariesDirectory)
     displayName: Setup vcpkg
 
+  - task: Cache@2
+    inputs:
+      key: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
+      restoreKeys: ccache | "$(CCACHE_VERSION)" | "$(Agent.OS)" | "${{ parameters.displayName }}"
+      path: $(CCACHE_DIR)
+      cacheHitVar: CCACHE_RESTORED
+    displayName: Restore ccache
+
+  - pwsh: |
+      $(Build.SourcesDirectory)/.azure-pipelines/install-ccache.ps1 -Destination "$(CCACHE_DIR)/bin"
+      echo "##vso[task.prependpath]$(CCACHE_DIR)/bin"
+    displayName: Install CCache
+    condition: ne(variables.CCACHE_RESTORED, 'true')
+
   - pwsh: |
-      $(Build.SourcesDirectory)\scripts\shell\enter_vs_dev_shell.ps1
-      $(Build.BinariesDirectory)\vcpkg\vcpkg.exe integrate install
+      $(Build.SourcesDirectory)/scripts/shell/enter_vs_dev_shell.ps1
+      $(Build.BinariesDirectory)/vcpkg/vcpkg.exe integrate install
       mkdir build
       cd build
       cmake .. `
         -GNinja `
-        -DCMAKE_MAKE_PROGRAM=$(Build.BinariesDirectory)\ninja.exe `
+        -DCMAKE_MAKE_PROGRAM=$(Build.BinariesDirectory)/ninja.exe `
         -DCMAKE_BUILD_TYPE=Release `
         -DTESTS_ENABLED=ON `
-        -DCMAKE_TOOLCHAIN_FILE=$(Build.BinariesDirectory)\vcpkg\scripts\buildsystems\vcpkg.cmake `
+        -DCMAKE_TOOLCHAIN_FILE=$(Build.BinariesDirectory)/vcpkg/scripts/buildsystems/vcpkg.cmake `
         -DVCPKG_TARGET_TRIPLET=x64-windows
-      $(Build.BinariesDirectory)\ninja.exe
+      $(Build.BinariesDirectory)/ninja.exe
     workingDirectory: $(Build.SourcesDirectory)
     displayName: Configure and build
 
   - pwsh: |
-      $(Build.BinariesDirectory)\vcpkg\vcpkg.exe integrate install
+      $(Build.BinariesDirectory)/vcpkg/vcpkg.exe integrate install
       ctest -E "(feature/sift_test)|(util/opengl_utils_test)|(mvs/gpu_mat_test)"
       $tests_pass=$LastExitCode
       if ($tests_pass -ne 0) {
@@ -58,3 +74,9 @@ jobs:
       exit $tests_pass
     workingDirectory: $(Build.SourcesDirectory)/build
     displayName: Run tests
+
+  - pwsh: |
+      ccache --show-stats
+      ccache --evict-older-than 1d
+      ccache --show-stats
+    displayName: Cleanup ccache
diff --git a/.azure-pipelines/install-ccache.ps1 b/.azure-pipelines/install-ccache.ps1
@@ -0,0 +1,38 @@
+[CmdletBinding()]
+param (
+    [Parameter(Mandatory = $true)]
+    [string] $Destination
+)
+
+$version = "4.8"
+$folder = "ccache-$version-windows-x86_64"
+$url = "https://github.com/ccache/ccache/releases/download/v$version/$folder.zip"
+$expectedSha256 = "A2B3BAB4BB8318FFC5B3E4074DC25636258BC7E4B51261F7D9BEF8127FDA8309"
+
+$ErrorActionPreference = "Stop"
+
+try {
+    New-Item -Path "$Destination" -ItemType Container -ErrorAction SilentlyContinue
+
+    Write-Host "Download CCache"
+    $zipFilePath = Join-Path "$env:TEMP" "$folder.zip"
+    Invoke-WebRequest -Uri $url -UseBasicParsing -OutFile "$zipFilePath" -MaximumRetryCount 3
+
+    $hash = Get-FileHash $zipFilePath -Algorithm "sha256"
+    if ($hash.Hash -ne $expectedSha256) {
+        throw "File $Path hash $hash.Hash did not match expected hash $expectedHash"
+    }
+
+    Write-Host "Unzip CCache"
+    Expand-Archive -Path "$zipFilePath" -DestinationPath "$env:TEMP"
+
+    Write-Host "Move CCache"
+    Move-Item "$env:TEMP/$folder/ccache.exe" "$Destination"
+    Remove-Item "$zipFilePath"
+    Remove-Item -Recurse "$env:TEMP/$folder"
+}
+catch {
+    Write-Host "Installation failed with an error"
+    $_.Exception | Format-List
+    exit -1
+}
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -296,7 +296,7 @@ else()
     message(STATUS "Disabling OpenGL support")
 endif()
 
-if(CCACHE_ENABLED AND NOT "${CMAKE_BUILD_TYPE}" STREQUAL "ClangTidy")
+if(CCACHE_ENABLED)
     find_program(CCACHE ccache)
     if(CCACHE)
         message(STATUS "Enabling ccache support")

diff --git a/cmake/CMakeHelper.cmake b/cmake/CMakeHelper.cmake
@@ -129,6 +129,9 @@ macro(COLMAP_ADD_CUDA_LIBRARY TARGET_NAME)
     target_link_libraries(${TARGET_NAME} CUDA::cudart CUDA::curand)
     set_target_properties(${TARGET_NAME} PROPERTIES FOLDER
         ${COLMAP_TARGETS_ROOT_FOLDER}/${FOLDER_NAME})
+    if(CLANG_TIDY_EXE)
+        set_target_properties(${TARGET_NAME} PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_EXE};-header-filter=.*")
+    endif()
     install(TARGETS ${TARGET_NAME} DESTINATION lib/colmap/)
 endmacro(COLMAP_ADD_CUDA_LIBRARY)
 
@@ -177,6 +180,9 @@ macro(COLMAP_ADD_CUDA_TEST TARGET_NAME)
         add_executable(${TARGET_NAME} ${ARGN})
         set_target_properties(${TARGET_NAME} PROPERTIES FOLDER
             ${COLMAP_TARGETS_ROOT_FOLDER}/${FOLDER_NAME})
+        if(CLANG_TIDY_EXE)
+            set_target_properties(${TARGET_NAME} PROPERTIES CXX_CLANG_TIDY "${CLANG_TIDY_EXE};-header-filter=.*")
+        endif()
         target_link_libraries(${TARGET_NAME} colmap
                               ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY})
         add_test("${FOLDER_NAME}/${TARGET_NAME}" ${TARGET_NAME})