CSHARP-4765: CI script to generate Driver's dev-package (#1169)

2 years ago · cb904135dc
12 changed files with 224 additions and 6 deletions
--- a/.editorconfig
+++ b/.editorconfig
@ -187,5 +187,6 @@ indent_size = 2
 # Shell scripts
 [*.sh]
 end_of_line = lf
+indent_size = 2
 [*.{cmd, bat}]
 end_of_line = crlf
--- a/.gitignore
+++ b/.gitignore
@ -48,6 +48,7 @@ src/packages

 # Nupkg artifacts
 *.nupkg
+*.snupkg

 # NCrunch artifacts
 *.ncrunch*
--- a/evergreen/evergreen.yml
+++ b/evergreen/evergreen.yml
@ -31,10 +31,10 @@ functions:
    - command: git.get_project
      params:
        directory: mongo-csharp-driver
-    # Applies the subitted patch, if any
+    # Applies the submitted patch, if any
    # Deprecated. Should be removed. But still needed for certain agents (ZAP)
    - command: git.apply_patch
-    # Make an evergreen exapanstion file with dynamic values
+    # Make an evergreen expansion file with dynamic values
    - command: shell.exec
      params:
        working_dir: mongo-csharp-driver
@ -46,6 +46,11 @@ functions:
              CURRENT_VERSION=latest
           fi

+           if [ "${PACKAGE_TARGET}" != "" ]; then
+              PACKAGE_VERSION=$(PACKAGE_TARGET="${PACKAGE_TARGET}" sh ./evergreen/packages-version.sh)
+           fi
+
+           export DOTNET_SDK_PATH="$(pwd)/../.dotnet"
           export DRIVERS_TOOLS="$(pwd)/../drivers-tools"

           if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
@ -68,6 +73,8 @@ functions:
           MONGODB_BINARIES: "$MONGODB_BINARIES"
           UPLOAD_BUCKET: "$UPLOAD_BUCKET"
           PROJECT_DIRECTORY: "$PROJECT_DIRECTORY"
+           PACKAGE_VERSION: "$PACKAGE_VERSION"
+           DOTNET_SDK_PATH: "$DOTNET_SDK_PATH"
           PREPARE_SHELL: |
              set -o errexit
              set -o xtrace
@ -76,14 +83,16 @@ functions:
              export MONGODB_BINARIES="$MONGODB_BINARIES"
              export UPLOAD_BUCKET="$UPLOAD_BUCKET"
              export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
-
+              export PACKAGE_VERSION="$PACKAGE_VERSION"
              export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
-              export PATH="$MONGODB_BINARIES:$PATH"
+              export PATH="$DOTNET_SDK_PATH:$MONGODB_BINARIES:$PATH"
              export PROJECT="${project}"
           EOT
           # See what we've done
           cat expansion.yml

+           # Do not output expansion.yml contents after this point
+
           # Add CSFLE variables that shouldn't be output to the logs
           cat <<EOT >> expansion.yml
           PREPARE_CSFLE: |
@ -97,13 +106,19 @@ functions:
              export FLE_GCP_PRIVATE_KEY=${FLE_GCP_PRIVATE_KEY}
              set -o xtrace # Enable tracing.
           EOT
-           # Do not output expansion.yml contents after this point

    # Load the expansion file to make an evergreen variable with the current unique version
    - command: expansions.update
      params:
        file: mongo-csharp-driver/expansion.yml

+  install-dotnet:
+    - command: shell.exec
+      params:
+        script: |
+          ${PREPARE_SHELL}
+          OS=${OS} sh ${PROJECT_DIRECTORY}/evergreen/install-dotnet.sh
+
  prepare-resources:
    - command: shell.exec
      params:
@ -1023,6 +1038,61 @@ functions:
          ${PREPARE_SHELL}
          echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created" } ]}' > ${PROJECT_DIRECTORY}/test-results.json

+  packages-pack:
+    - command: shell.exec
+      params:
+        working_dir: mongo-csharp-driver
+        script: |
+          ${PREPARE_SHELL}
+          . ./evergreen/packages-pack.sh
+
+  packages-push:
+    - command: shell.exec
+      params:
+        working_dir: mongo-csharp-driver
+        env:
+          PACKAGES_SOURCE: ${PACKAGES_SOURCE}
+          PACKAGES_SOURCE_KEY: ${PACKAGES_SOURCE_KEY}
+        script: |
+          ${PREPARE_SHELL}
+          . ./evergreen/packages-push.sh
+
+  upload-package:
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: ${PROJECT_DIRECTORY}/build/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg
+        remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg
+        bucket: mciuploads
+        permissions: public-read
+        content_type: ${content_type|application/octet-stream}
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: ${PROJECT_DIRECTORY}/build/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg
+        remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg
+        bucket: mciuploads
+        permissions: public-read
+        content_type: ${content_type|application/octet-stream}
+
+  download-package:
+    - command: s3.get
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: ${PROJECT_DIRECTORY}/build/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg
+        remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.nupkg
+        bucket: mciuploads
+    - command: s3.get
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: ${PROJECT_DIRECTORY}/build/nuget/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg
+        remote_file: ${UPLOAD_BUCKET}/${revision}/${PACKAGE_ID}.${PACKAGE_VERSION}.snupkg
+        bucket: mciuploads
+
 pre:
  - func: fetch-source
  - func: prepare-resources
@ -1040,7 +1110,6 @@ post:
  - func: cleanup

 tasks:
-
    - name: test-net472
      commands:
        - func: bootstrap-mongo-orchestration
@ -1676,6 +1745,47 @@ tasks:
              export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
              GCPKMS_CMD="MONGODB_URI='mongodb://localhost:27017' ./evergreen/run-csfle-gcp-tests.sh" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh

+    - name: packages-create
+      commands:
+      - func: packages-pack
+      - func: upload-package
+        vars:
+          PACKAGE_ID: "MongoDB.Bson"
+      - func: upload-package
+        vars:
+          PACKAGE_ID: "MongoDB.Driver"
+      - func: upload-package
+        vars:
+          PACKAGE_ID: "MongoDB.Driver.Core"
+      - func: upload-package
+        vars:
+          PACKAGE_ID: "MongoDB.Driver.GridFS"
+      - func: upload-package
+        vars:
+          PACKAGE_ID: "mongocsharpdriver"
+
+    - name: packages-push
+      commands:
+        - func: download-package
+          vars:
+            PACKAGE_ID: "MongoDB.Bson"
+        - func: download-package
+          vars:
+            PACKAGE_ID: "MongoDB.Driver"
+        - func: download-package
+          vars:
+            PACKAGE_ID: "MongoDB.Driver.Core"
+        - func: download-package
+          vars:
+            PACKAGE_ID: "MongoDB.Driver.GridFS"
+        - func: download-package
+          vars:
+            PACKAGE_ID: "mongocsharpdriver"
+        - func: packages-push
+          vars:
+            PACKAGES_SOURCE: "https://api.nuget.org/v3/index.json"
+            PACKAGES_SOURCE_KEY: ${nuget_api_key}
+
 axes:
  - id: version
    display_name: MongoDB Version
@ -1897,6 +2007,19 @@ task_groups:
    tasks:
      - test-csfle-with-gcp-kms

+  - name: dev-package
+    setup_task:
+      - func: fetch-source
+        vars:
+          PACKAGE_TARGET: "dev"
+      - func: prepare-resources
+      - func: fix-absolute-paths
+      - func: make-files-executable
+      - func: install-dotnet
+    tasks:
+      - packages-create
+      - packages-push
+
 buildvariants:

 - matrix_name: "secure-tests"
@ -2200,3 +2323,11 @@ buildvariants:
    - name: test-smoke-tests-netcoreapp31
    - name: test-smoke-tests-net50
    - name: test-smoke-tests-net60
+
+- name: dev-package-pack
+  git_tag_only: true
+  display_name: "Dev Package Pack"
+  run_on: ubuntu1804-small
+  tags: ["dev-package"]
+  tasks:
+    - name: dev-package
--- a/evergreen/install-dotnet.sh
+++ b/evergreen/install-dotnet.sh
@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+set -o errexit  # Exit the script with error if any of the commands fail
+
+DOTNET_SDK_PATH=./.dotnet
+mkdir -p "$DOTNET_SDK_PATH"
+
+if [[ $OS =~ [Ww]indows.* ]]; then
+  echo "Downloading Windows .NET SDK installer into $DOTNET_SDK_PATH folder..."
+  curl -Lfo "$DOTNET_SDK_PATH"/dotnet-install.ps1 https://dot.net/v1/dotnet-install.ps1
+  echo "Installing .NET LTS SDK..."
+  powershell.exe "$DOTNET_SDK_PATH"/dotnet-install.ps1 -Channel 6.0 -InstallDir "$DOTNET_SDK_PATH" -NoPath
+else
+  echo "Downloading .NET SDK installer into $DOTNET_SDK_PATH folder..."
+  curl -Lfo "$DOTNET_SDK_PATH"/dotnet-install.sh https://dot.net/v1/dotnet-install.sh
+  echo "Installing .NET LTS SDK..."
+  bash "$DOTNET_SDK_PATH"/dotnet-install.sh --channel 6.0 --install-dir "$DOTNET_SDK_PATH" --no-path
+fi
--- a/evergreen/packages-pack.sh
+++ b/evergreen/packages-pack.sh
@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -o errexit  # Exit the script with error if any of the commands fail
+
+if [ -z "$PACKAGE_VERSION" ]; then
+  PACKAGE_VERSION=$(sh ./evergreen/packages-version.sh)
+  echo Calculated PACKAGE_VERSION value: "$PACKAGE_VERSION"
+fi
+
+echo Creating nuget package...
+
+dotnet clean ./CSharpDriver.sln
+dotnet pack ./CSharpDriver.sln -o ./build/nuget -c Release -p:Version="$PACKAGE_VERSION" --include-symbols -p:SymbolPackageFormat=snupkg
--- a/evergreen/packages-push.sh
+++ b/evergreen/packages-push.sh
@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -o errexit  # Exit the script with error if any of the commands fail
+set +o xtrace # Disable tracing.
+
+if [ -z "$PACKAGES_SOURCE" ]; then
+  echo "PACKAGES_SOURCE variable should be set"
+  exit 1
+fi
+
+if [ -z "$PACKAGES_SOURCE_KEY" ]; then
+  echo "PACKAGES_SOURCE_KEY variable should be set"
+  exit 1
+fi
+
+if [ -z "$PACKAGE_VERSION" ]; then
+  echo "PACKAGE_VERSION variable should be set"
+  exit 1
+fi
+
+dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/MongoDB.Bson."$PACKAGE_VERSION".nupkg
+dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/MongoDB.Driver.Core."$PACKAGE_VERSION".nupkg
+dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/MongoDB.Driver."$PACKAGE_VERSION".nupkg
+dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/MongoDB.Driver.GridFS."$PACKAGE_VERSION".nupkg
+dotnet nuget push --source "$PACKAGES_SOURCE" --api-key "$PACKAGES_SOURCE_KEY" ./build/nuget/mongocsharpdriver."$PACKAGE_VERSION".nupkg
+
+
--- a/evergreen/packages-version.sh
+++ b/evergreen/packages-version.sh
@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -o errexit  # Exit the script with error if any of the commands fail
+
+if [ -z "$PACKAGE_TARGET" ]; then
+  # Use production release tag if nothing was passed
+  PACKAGE_TARGET="release"
+fi
+
+if [ "${PACKAGE_TARGET}" = "dev" ]; then
+  PACKAGE_VERSION_MATCH="v[0-9]*.[0-9]*.[0-9]*-dev[0-9]*"
+  PACKAGE_VERSION_EXCLUDE=""
+elif [ "${PACKAGE_TARGET}" = "release" ]; then
+  PACKAGE_VERSION_MATCH="v[0-9]*.[0-9]*.[0-9]*"
+  PACKAGE_VERSION_EXCLUDE="*-dev[0-9]*"
+else
+  echo "Unexpected value of PACKAGE_TARGET: ${PACKAGE_TARGET}"
+fi
+
+PACKAGE_VERSION=$(git describe --tags --abbrev=0 --match="${PACKAGE_VERSION_MATCH}" --exclude="${PACKAGE_VERSION_EXCLUDE}")
+PACKAGE_VERSION=$(echo $PACKAGE_VERSION | cut -c 2-)
+echo "$PACKAGE_VERSION"
--- a/src/MongoDB.Bson/MongoDB.Bson.csproj
+++ b/src/MongoDB.Bson/MongoDB.Bson.csproj
@ -7,9 +7,11 @@
  <PropertyGroup>
    <AssemblyTitle>MongoDB.Bson</AssemblyTitle>
    <Product>MongoDB.Bson</Product>
+    <PackageId>MongoDB.Bson</PackageId>
    <Description>Official MongoDB supported BSON library. See https://www.mongodb.com/docs/drivers/csharp/ for more details.</Description>
    <PackageDescription>MongoDB's Official Bson Library.</PackageDescription>
    <PackageTags>$(PackageTags);bson</PackageTags>
+    <IsPackable>true</IsPackable>
  </PropertyGroup>

  <ItemGroup>
--- a/src/MongoDB.Driver.Core/MongoDB.Driver.Core.csproj
+++ b/src/MongoDB.Driver.Core/MongoDB.Driver.Core.csproj
@ -6,8 +6,10 @@
  <PropertyGroup>
    <AssemblyTitle>MongoDB.Driver.Core</AssemblyTitle>
    <Product>MongoDB.Driver.Core</Product>
+    <PackageId>MongoDB.Driver.Core</PackageId>
    <Description>Official MongoDB supported Driver Core library. See https://www.mongodb.com/docs/drivers/csharp/ for more details.</Description>
    <PackageDescription>Core Component of the Official MongoDB .NET Driver.</PackageDescription>
+    <IsPackable>true</IsPackable>
  </PropertyGroup>

  <PropertyGroup>
--- a/src/MongoDB.Driver.GridFS/MongoDB.Driver.GridFS.csproj
+++ b/src/MongoDB.Driver.GridFS/MongoDB.Driver.GridFS.csproj
@ -7,9 +7,11 @@
  <PropertyGroup>
    <AssemblyTitle>MongoDB.Driver.GridFS</AssemblyTitle>
    <Product>MongoDB.Driver.GridFS</Product>
+    <PackageId>MongoDB.Driver.GridFS</PackageId>
    <Description>Official MongoDB supported driver for MongoDB GridFS implementation. See https://www.mongodb.com/docs/drivers/csharp/ for more details.</Description>
    <PackageDescription>GridFS Component of the Official MongoDB .NET Driver.</PackageDescription>
    <PackageTags>$(PackageTags);gridfs</PackageTags>
+    <IsPackable>true</IsPackable>
  </PropertyGroup>

  <ItemGroup>
--- a/src/MongoDB.Driver.Legacy/MongoDB.Driver.Legacy.csproj
+++ b/src/MongoDB.Driver.Legacy/MongoDB.Driver.Legacy.csproj
@ -11,6 +11,7 @@
    <PackageId>mongocsharpdriver</PackageId>
    <PackageDescription>This package contains the legacy driver. The new driver's package name is MongoDB.Driver</PackageDescription>
    <PackageTags></PackageTags>
+    <IsPackable>true</IsPackable>
  </PropertyGroup>

  <PropertyGroup>
--- a/src/MongoDB.Driver/MongoDB.Driver.csproj
+++ b/src/MongoDB.Driver/MongoDB.Driver.csproj
@ -7,8 +7,10 @@
  <PropertyGroup>
    <AssemblyTitle>MongoDB.Driver</AssemblyTitle>
    <Product>MongoDB.Driver</Product>
+    <PackageId>MongoDB.Driver</PackageId>
    <Description>Official MongoDB supported driver for MongoDB. See https://www.mongodb.com/docs/drivers/csharp/ for more details.</Description>
    <PackageDescription>Official .NET driver for MongoDB.</PackageDescription>
+    <IsPackable>true</IsPackable>
  </PropertyGroup>

  <ItemGroup>