Merge branch 'refactor/deploy' into 'master'

Refactor deployment

See merge request !35

shared_scripts.yml

 #-------------------------------------------------------------------------------
-# Copyright (c) 2020 RWTH Aachen University, Germany,
+# Copyright (c) 2022 RWTH Aachen University, Germany,
 # Virtual Reality & Immersive Visualisation Group.
 #-------------------------------------------------------------------------------
 
@@ -10,7 +10,7 @@ variables:
   GIT_CLEAN_FLAGS: none
   GIT_DEPTH: "1"
   NUMBER_OF_OLD_VERSIONS: 3         # number of old revisions to keep after the deploy
-  CLUSTER_ADDRESS: "login.hpc.itc.rwth-aachen.de"
+  CLUSTER_ADDRESS: "copy.hpc.itc.rwth-aachen.de"
   SHARED_META_URL: "https://git-ce.rwth-aachen.de/vr-vis/VR-Group/unreal-development/unreal-ci/-/raw/master/shared.meta"
 
 ##################################Generate Project##################################
@@ -69,32 +69,21 @@ variables:
     - $env:Path="$env:PREPEND_PATH;$env:Path;$env:APPEND_PATH"
     - $env:PROJECT_NAME=(Get-ChildItem *.uproject).BaseName
     - . "$env:UE_ENGINE_FOLDER\Build\BatchFiles\RunUAT.bat" BuildCookRun -Project="$env:CI_PROJECT_DIR/$env:PROJECT_NAME.uproject" -WaitMutex -NoP4 -TargetPlatform=Win64 -ClientConfig="$env:CLIENT_CONFIG" -Cook -allmaps -Build -Stage -Pak -Iterate -utf8output -Archive -ArchiveDirectory="$env:CI_PROJECT_DIR/Release/$env:CI_PROJECT_NAME" -Rocket -Package ($env:EXTRA_ARGS); $env:EXIT_CODE=$LASTEXITCODE
-        #
-        # Copy metadata and replace the shared part with a downloaded shared part
-        # Define some variables to use in shared part and copy them in via second envsubst
-        #
-        - if($env:EXIT_CODE -eq 0){
-        -   if(Test-Path $env:CI_PROJECT_DIR/launcher.meta){
-        -      $env:SHARED_META=$((Invoke-WebRequest $env:SHARED_META_URL -UseBasicParsing).Content | head -n -1 | tail -n +2 | Out-String)
-        -      $env:CI_DEPLOY_TIME=$(Get-Date -UFormat +%Y_%m_%d_%H_%M)
-        -      $env:CLEANED_COMMIT_MSG=$(echo $env:CI_COMMIT_MESSAGE | python -c "import json,sys; print(json.dumps(sys.stdin.read()).strip('\`"'))")
-        -      cat $env:CI_PROJECT_DIR/launcher.meta | sed '/CI-DATA-ENTRIES/c\${SHARED_META}' | envsubst > $env:CI_PROJECT_DIR/Release/$env:CI_PROJECT_NAME/WindowsNoEditor/launcher.meta.tmp
-        -      cat $env:CI_PROJECT_DIR/Release/$env:CI_PROJECT_NAME/WindowsNoEditor/launcher.meta.tmp | envsubst > $env:CI_PROJECT_DIR/Release/$env:CI_PROJECT_NAME/WindowsNoEditor/launcher.meta
-        -      rm -Force $env:CI_PROJECT_DIR/Release/$env:CI_PROJECT_NAME/WindowsNoEditor/launcher.meta.tmp
-            }
-        -   cp -ErrorAction Ignore "$env:CI_PROJECT_DIR/Saved/AutoScreenshot.png" "$env:CI_PROJECT_DIR/Release/$env:CI_PROJECT_NAME/WindowsNoEditor/thumbnail.png" ; $true | out-null
-        -   cp -ErrorAction Ignore "$env:CI_PROJECT_DIR/$env:PROJECT_NAME.png" "$env:CI_PROJECT_DIR/Release/$env:CI_PROJECT_NAME/WindowsNoEditor/thumbnail.png" ; $true | out-null
-          } else {
-        - echo "*********************************************`nThe build failed. You can find all output logs in the artifacts.`n*********************************************"
-          }
-        - mkdir -f "$env:CI_PROJECT_DIR/Release/$env:CI_PROJECT_NAME/WindowsNoEditor/BuildLogs/" | out-null
-        - cp -ErrorAction Ignore -R "$env:ADDITIONAL_LOGS/*" "$env:CI_PROJECT_DIR/Release/$env:CI_PROJECT_NAME/WindowsNoEditor/BuildLogs/" ; $true | out-null
+    # Copy build logs
+    - mkdir -f "$env:CI_PROJECT_DIR/Release/$env:CI_PROJECT_NAME/BuildLogs/" | out-null
+    - cp -ErrorAction Ignore -R "$env:ADDITIONAL_LOGS/*" "$env:CI_PROJECT_DIR/Release/$env:CI_PROJECT_NAME/BuildLogs/" ; $true | out-null
+    # Rename thumbnail if it exists
+    - mv -ErrorAction Ignore ${PROJECT_NAME}.png thumbnail.png ; $true | out-null
+    # Exit with exit code of the build
     - exit $env:EXIT_CODE
   timeout: 3h
   artifacts:
     name: "Packaged_Windows_$env:CI_PROJECT_NAME"
     paths:
       - Release/$env:CI_PROJECT_NAME/
+      - Saved/AutoScreenshot.png
+      - thumbnail.png
+      - launcher.meta
     expire_in: 3 days
     when: always
         
@@ -118,75 +107,74 @@ variables:
     - export PROJECT_NAME=`basename $(find *.uproject) .uproject`
     - EXIT_CODE=0
     - (${UE_ENGINE_FOLDER}/Build/BatchFiles/RunUAT.sh BuildCookRun -project="${CI_PROJECT_DIR}/${PROJECT_NAME}.uproject" -WaitMutex -nop4 -targetplatform=Linux -clientconfig=${CLIENT_CONFIG} -cook -allmaps -build -stage -pak -iterate -utf8output -archive -archivedirectory="${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}" -rocket -package ${EXTRA_ARGS}) || EXIT_CODE=$?
-        #
-        # Copy metadata and replace the shared part with a downloaded shared part
-        # Define some variables to use in shared part and copy them in via second envsubst
-        #
-        - if [ ${EXIT_CODE} -eq 0 ] ; then
-        -   if [ -f ${CI_PROJECT_DIR}/launcher.meta ] ; then
-        -     export SHARED_META=$(wget -qO- ${SHARED_META_URL} | head -n -1 | tail -n +2)
-        -     export CI_DEPLOY_TIME=$(date +%Y_%m_%d_%H_%M)
-        -     export CLEANED_COMMIT_MSG=$(echo ${CI_COMMIT_MESSAGE} | python -c "import json,sys; print(json.dumps(sys.stdin.read()).strip('\"'))")
-        -     cat ${CI_PROJECT_DIR}/launcher.meta | sed '/CI-DATA-ENTRIES/c\${SHARED_META}' | envsubst > ${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/LinuxNoEditor/launcher.meta.tmp
-        -     cat ${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/LinuxNoEditor/launcher.meta.tmp | envsubst > ${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/LinuxNoEditor/launcher.meta
-        -     rm -f ${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/LinuxNoEditor/launcher.meta.tmp
-        -   fi
-        -   cp "${CI_PROJECT_DIR}/Saved/AutoScreenshot.png" "${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/LinuxNoEditor/thumbnail.png" 2> /dev/null || true
-        -   cp "${CI_PROJECT_DIR}/${PROJECT_NAME}.png" "${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/LinuxNoEditor/thumbnail.png" 2> /dev/null || true
-        - else
-        -   echo -e "*********************************************The build failed. You can find all output logs in the artifacts.*********************************************"
-        - fi
-        - mkdir -p "${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/LinuxNoEditor/BuildLogs/"
-        - cp -R "${ADDITIONAL_LOGS}/"* "${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/LinuxNoEditor/BuildLogs/" || true
+    # Copy build logs
+    - mkdir -p "${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/BuildLogs/"
+    - cp -R "${ADDITIONAL_LOGS}/"* "${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/BuildLogs/" || true
+    # Rename thumbnail if it exists
+    - mv ${PROJECT_NAME}.png thumbnail.png || true
+    # Exit with exit code of the build
     - exit ${EXIT_CODE}
   timeout: 3h
   artifacts:
     name: "Packaged_Linux_${CI_PROJECT_NAME}"
     paths:
       - Release/${CI_PROJECT_NAME}/
+      - Saved/AutoScreenshot.png
+      - thumbnail.png
+      - launcher.meta
     expire_in: 3 days
     when: always
         
 ##################################Deploy##################################
 
-.Deploy_VRDev_:
+.Deploy_General_:
   stage: deploy
   tags:
     - linux
-        - networkdeploy
+    - docker-executor
+  image:
+    name: registry.git-ce.rwth-aachen.de/vr-vis/vr-group/alpine-ci-container/alpine-ci:latest
   allow_failure: true
+  variables:
+    SYNC_KEY: ""
+    USER: ""
+    CREATE_LAUNCHER_META: "true"
   script:
-        # Get all previous builds from network
-        - rsync -e "ssh -i $HOME/.ssh/cluster_access_vrdev" --archive --info=progress2 --human-readable --no-inc-recursive --delete --exclude=*.11 --exclude 'Thumbs.db' --chmod=0770 vrdev@${CLUSTER_ADDRESS}:/ ${VRDEV_UNREAL_DEPLOY_FOLDER}/ || true
-        # Copy build into the previous builds folder and overwrite old folder
-        - CURRENT_VRDEV_FOLDER=${VRDEV_UNREAL_DEPLOY_FOLDER}/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/${CI_PIPELINE_IID}
-        - find ${CURRENT_VRDEV_FOLDER} -delete &> /dev/null || true
-        - mkdir -p ${CURRENT_VRDEV_FOLDER}/
-        - mv -f ${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/WindowsNoEditor/ ${CURRENT_VRDEV_FOLDER}/
-        # Remove all old versions except the last $NUMBER_OF_OLD_VERSIONS versions
-        - find ${VRDEV_UNREAL_DEPLOY_FOLDER}/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/ -mindepth 1 -maxdepth 1 -type d -name "[[:digit:]]*" | sort -V -r | sed -e "1,${NUMBER_OF_OLD_VERSIONS}d" | xargs -d '\n' rm -rf
-        - rsync -e "ssh -i $HOME/.ssh/cluster_access_vrdev" --archive --info=progress2 --human-readable --no-inc-recursive --delete --relative --chown=vrdev:av00 --chmod=0770 --exclude "${CURRENT_VRDEV_FOLDER}/WindowsNoEditor/launcher.meta" ${VRDEV_UNREAL_DEPLOY_FOLDER}/./${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/ vrdev@${CLUSTER_ADDRESS}:/
-        - if [ -f ${CURRENT_VRDEV_FOLDER}/WindowsNoEditor/launcher.meta ] ; then
-        -    rsync -e "ssh -i $HOME/.ssh/cluster_access_vrdev" --archive --info=progress2 --human-readable --no-inc-recursive --delete --relative --chown=vrdev:av00 --chmod=0770 ${VRDEV_UNREAL_DEPLOY_FOLDER}/./${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/${CI_PIPELINE_IID}/WindowsNoEditor/launcher.meta vrdev@${CLUSTER_ADDRESS}:/
+    # Change access rights for SSH key to use it later on
+    - chmod 600 ${SYNC_KEY}
+    # Find out if it was Windows, WindowsNoEditor, Linux or LinuxNoEditor
+    - export PLATFORM_NAME=$(find ${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/ -mindepth 1 -maxdepth 1 -type d -name "*Windows*" -or -name "*Linux*" | head -1 | xargs basename)
+    # Copy Thumbnails if they exist
+    - cp "${CI_PROJECT_DIR}/Saved/AutoScreenshot.png" "${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/${PLATFORM_NAME}/thumbnail.png" 2> /dev/null || true
+    - cp "${CI_PROJECT_DIR}/thumbnail.png" "${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/${PLATFORM_NAME}/thumbnail.png" 2> /dev/null || true
+    # RSync to Cluster
+    - rsync -e "ssh -i ${SYNC_KEY}" --archive --info=progress2 --mkpath --human-readable --no-inc-recursive --chown=${USER}:av00 --chmod=0770 ${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/ ${USER}@${CLUSTER_ADDRESS}:/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/${CI_PIPELINE_IID}/
+    # Generate launcher.meta
+    - if [ -f ${CI_PROJECT_DIR}/launcher.meta ] && [[ "${CREATE_LAUNCHER_META}" == "true" ]] ; then
+      # Define some variables which will be used in the generated launcher.meta
+    -   export CI_DEPLOY_TIME=$(date +%Y_%m_%d_%H_%M)
+    -   export CLEANED_COMMIT_MSG=$(echo ${CI_COMMIT_MESSAGE} | jq -cRM | tail -c +2 | head -c -2)
+        # Substitute the values into place
+    -   export SHARED_META=$(wget -qO- ${SHARED_META_URL} | head -n -1 | tail -n +2)
+    -   cat ${CI_PROJECT_DIR}/launcher.meta | sed '/CI-DATA-ENTRIES/c\${SHARED_META}' | envsubst > ${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/${PLATFORM_NAME}/launcher.meta.tmp
+    -   cat ${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/${PLATFORM_NAME}/launcher.meta.tmp | envsubst > ${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/${PLATFORM_NAME}/launcher.meta
+    -   rm -f ${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/${PLATFORM_NAME}/launcher.meta.tmp
+
+        # RSync launcher.meta last to allow launch only after sync is done
+    -   rsync -e "ssh -i ${SYNC_KEY}" --archive --info=progress2 --mkpath --human-readable --no-inc-recursive --chown=${USER}:av00 --chmod=0770 ${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/${PLATFORM_NAME}/launcher.meta ${USER}@${CLUSTER_ADDRESS}:/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/${CI_PIPELINE_IID}/${PLATFORM_NAME}/
     - fi
     
+
+.Deploy_VRDev_:
+  extends: .Deploy_General_
+  variables:
+    SYNC_KEY: "${SSH_KEY_RSYNC_VRDEV}"
+    USER: "vrdev"
+    CREATE_LAUNCHER_META: "false"
+        
 .Deploy_CAVE_:
-    stage: deploy
-    tags:
-        - linux
-        - networkdeploy
-    allow_failure: true
-    script:
-        # Get all previous builds from network
-        - rsync -e "ssh -i $HOME/.ssh/cluster_access_av006de" --archive --info=progress2 --human-readable --no-inc-recursive --delete --exclude=*.11 --exclude 'Thumbs.db' --chmod=0770 av006de@${CLUSTER_ADDRESS}:/ ${CAVE_UNREAL_DEPLOY_FOLDER}/ || true
-        # Copy build into the previous builds folder and overwrite old folder
-        - CURRENT_CAVE_FOLDER=${CAVE_UNREAL_DEPLOY_FOLDER}/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/${CI_PIPELINE_IID}
-        - find ${CURRENT_CAVE_FOLDER} -delete &> /dev/null || true
-        - mkdir -p ${CURRENT_CAVE_FOLDER}/
-        - mv -f ${CI_PROJECT_DIR}/Release/${CI_PROJECT_NAME}/LinuxNoEditor/ ${CURRENT_CAVE_FOLDER}/
-        # Remove all old versions except the last $NUMBER_OF_OLD_VERSIONS versions
-        - find ${CAVE_UNREAL_DEPLOY_FOLDER}/${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/ -mindepth 1 -maxdepth 1 -type d -name "[[:digit:]]*" | sort -V -r | sed -e "1,${NUMBER_OF_OLD_VERSIONS}d" | xargs -d '\n' rm -rf
-        - rsync -e "ssh -i $HOME/.ssh/cluster_access_av006de" --archive --info=progress2 --human-readable --no-inc-recursive --delete --relative --chown=av006de:av00 --chmod=0770 --exclude "${CURRENT_CAVE_FOLDER}/LinuxNoEditor/launcher.meta" ${CAVE_UNREAL_DEPLOY_FOLDER}/./${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/ av006de@${CLUSTER_ADDRESS}:/
-        - if [ -f ${CURRENT_CAVE_FOLDER}/LinuxNoEditor/launcher.meta ] ; then
-        -   rsync -e "ssh -i $HOME/.ssh/cluster_access_av006de" --archive --info=progress2 --human-readable --no-inc-recursive --delete --relative --chown=av006de:av00 --chmod=0770 ${CAVE_UNREAL_DEPLOY_FOLDER}/./${CI_PROJECT_NAME}/${CI_COMMIT_REF_NAME}/${CI_PIPELINE_IID}/LinuxNoEditor/launcher.meta av006de@${CLUSTER_ADDRESS}:/
-        - fi
+  extends: .Deploy_General_
+  variables:
+    SYNC_KEY: "${SSH_KEY_RSYNC_CAVE}"
+    USER: "av006de"
+    CREATE_LAUNCHER_META: "true"