Merge branch 'stable-23.10' into MCOL-4628-SET-DOUBLE_COL-TO-DECIMAL_COL-RETURNS-ERROR

Author: AestheticAkhmad

.drone.jsonnet

@@ -15,10 +15,16 @@ local platforms_arm = {
   "stable-23.10": ["rockylinux:8", "rockylinux:9", "debian:12", "ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"],
 };
 
+local rewrite_ubuntu_mirror = @"sed -i 's|//\\(us\\.\\)\\?archive\\.ubuntu\\.com|//us.archive.ubuntu.com|g' /etc/apt/sources.list || true; " +
+                              @"sed -i 's|//\\(us\\.\\)\\?archive\\.ubuntu\\.com|//us.archive.ubuntu.com|g' /etc/apt/sources.list.d/ubuntu.sources || true; " +
+                              "cat /etc/apt/sources.list.d/ubuntu.sources /etc/apt/sources.list | grep archive || true; ";
+
 local customEnvCommandsMap = {
   // 'clang-18': ['apt install -y clang-18', 'export CC=/usr/bin/clang-18', 'export CXX=/usr/bin/clang++-18'],
   "clang-20": [
-    "apt install -y wget curl lsb-release software-properties-common gnupg",
+    rewrite_ubuntu_mirror,
+    "apt-get clean && apt-get update",
+    "apt-get install -y wget curl lsb-release software-properties-common gnupg",
     "wget https://apt.llvm.org/llvm.sh",
     "bash llvm.sh 20",
     "export CC=/usr/bin/clang",
@@ -114,7 +120,7 @@ local rockylinux9_deps = "dnf install -y 'dnf-command(config-manager)' " +
 
 local rockylinux_common_deps = " && dnf install -y git lz4 lz4-devel cppunit-devel cmake3 boost-devel snappy-devel pcre2-devel";
 
-local deb_deps = "apt update && apt install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev";
+local deb_deps = rewrite_ubuntu_mirror + "apt-get clean && apt-get update && apt-get install --yes git libboost-all-dev libcppunit-dev libsnappy-dev cmake libpcre2-dev";
 
 local testPreparation(platform) =
   local platform_map = {
@@ -127,6 +133,10 @@ local testPreparation(platform) =
   };
   platform_map[platform];
 
+local make_clickable_link(link) = "echo -e '\\e]8;;" +  link + "\\e\\\\" +  link + "\\e]8;;\\e\\\\'";
+local echo_running_on = ["echo running on ${DRONE_STAGE_MACHINE}",
+      make_clickable_link("https://us-east-1.console.aws.amazon.com/ec2/home?region=us-east-1#Instances:search=:${DRONE_STAGE_MACHINE};v=3;$case=tags:true%5C,client:false;$regex=tags:false%5C,client:false;sort=desc:launchTime")];
+
 local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise", customBootstrapParams="", customBuildEnvCommandsMapKey="") = {
   local pkg_format = if (std.split(platform, ":")[0] == "rockylinux") then "rpm" else "deb",
   local init = if (pkg_format == "rpm") then "/usr/lib/systemd/systemd" else "systemd",
@@ -154,10 +164,12 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
   local server_remote = if (std.endsWith(server, "enterprise")) then "https://github.com/mariadb-corporation/MariaDBEnterprise" else "https://github.com/MariaDB/server",
 
   local sccache_arch = if (arch == "amd64") then "x86_64" else "aarch64",
-  local get_sccache = "echo getting sccache... && (apt update -y && apt install -y curl || yum install -y curl || true) " +
-                      "&& curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-" + sccache_arch + "-unknown-linux-musl.tar.gz " +
-                      "&& tar xzf sccache.tar.gz " +
-                      "&& install sccache*/sccache /usr/local/bin/ && echo sccache installed",
+  local get_sccache = ["echo getting sccache...",
+                      rewrite_ubuntu_mirror,
+                      "(apt-get clean && apt-get update -y && apt-get install -y curl || yum install -y curl || true)",
+                      "curl -L -o sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-" + sccache_arch + "-unknown-linux-musl.tar.gz &&",
+                      "tar xzf sccache.tar.gz",
+                      "install sccache*/sccache /usr/local/bin/ && echo sccache installed"],
 
   local pipeline = self,
 
@@ -175,14 +187,16 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
       AWS_SECRET_ACCESS_KEY: {
         from_secret: "aws_secret_access_key",
       },
+      AWS_REGION: "us-east-1",
+      AWS_DEFAULT_REGION: "us-east-1",
     },
     commands: [
       "ls " + result,
       '[ -z "$(ls -A "' + result + '")" ] && echo Nothing to publish! && exit 1',
 
       "aws s3 sync " + result + " s3://cspkg/" + branchp + eventp + "/" + server + "/" + arch + "/" + result + " --only-show-errors",
       'echo "Data uploaded to: ' + publish_pkg_url + '"',
-
+      make_clickable_link(publish_pkg_url),
       "rm -rf " + result + "/*",
     ],
   },
@@ -231,7 +245,7 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
   local getContainerName(stepname) = stepname + "$${DRONE_BUILD_NUMBER}",
 
   local installCmapi(containerName, pkg_format) =
-    if (pkg_format == "deb") then execInnerDocker('bash -c "apt update -y && apt install -y mariadb-columnstore-cmapi"', containerName)
+    if (pkg_format == "deb") then execInnerDocker('bash -c "apt-get clean && apt-get update -y && apt-get install -y mariadb-columnstore-cmapi"', containerName)
     else execInnerDocker('bash -c "yum update -y && yum install -y MariaDB-columnstore-cmapi"', containerName),
 
   local prepareTestStage(containerName, result, do_setup) =
@@ -606,8 +620,9 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
                SERVER_REMOTE: "${SERVER_REMOTE:-" + server_remote + "}",
                SERVER_SHA: "${SERVER_SHA:-" + server + "}",
              },
-             commands: [
-               "echo $$SERVER_REF",
+             commands: echo_running_on +
+             [
+              "echo $$SERVER_REF",
                "echo $$SERVER_REMOTE",
                "mkdir -p /mdb/" + builddir + " && cd /mdb/" + builddir,
                'git config --global url."https://github.com/".insteadOf [email protected]:',
@@ -626,15 +641,14 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
              volumes: [pipeline._volumes.mdb],
              environment: {
                DEBIAN_FRONTEND: "noninteractive",
-               DEB_BUILD_OPTIONS: "parallel=4",
-               DH_BUILD_DDEBS: "1",
-               BUILDPACKAGE_FLAGS: "-b",  // Save time and produce only binary packages, not source
                AWS_ACCESS_KEY_ID: {
                  from_secret: "aws_access_key_id",
                },
                AWS_SECRET_ACCESS_KEY: {
                  from_secret: "aws_secret_access_key",
                },
+               AWS_REGION: "us-east-1",
+               AWS_DEFAULT_REGION: "us-east-1",
                SCCACHE_BUCKET: "cs-sccache",
                SCCACHE_REGION: "us-east-1",
                SCCACHE_S3_USE_SSL: "true",
@@ -644,14 +658,14 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
              },
              commands: [
                          "mkdir /mdb/" + builddir + "/" + result,
-                         get_sccache,
                        ]
+                       + get_sccache
                        + customEnvCommands(customBuildEnvCommandsMapKey, builddir) +
                        [
-                         'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " +
+                        'bash -c "set -o pipefail && bash /mdb/' + builddir + "/storage/columnstore/columnstore/build/bootstrap_mcs.sh " +
                          "--build-type RelWithDebInfo " +
                          "--distro " + platform + " " +
-                         "--build-packages --sccache " +
+                         "--build-packages --install-deps --sccache " +
                          " " + customBootstrapParams +
                          " " + customBootstrapParamsForExisitingPipelines(platform) + " | " +
                          "/mdb/" + builddir + "/storage/columnstore/columnstore/build/ansi2txt.sh " +
@@ -728,19 +742,6 @@ local Pipeline(branch, platform, event, arch="amd64", server="10.6-enterprise",
          ] +
          [pipeline.publish("cmapi build")] +
          [pipeline.publish()] +
-         [
-           {
-             name: "publish pkg url",
-             depends_on: ["publish pkg"],
-             image: "alpine/git",
-             commands: [
-               "echo -e '\\e]8;;" + publish_pkg_url + "\\e\\\\" + publish_pkg_url + "\\e]8;;\\e\\\\'",
-               "echo 'for installation run:'",
-               "echo 'export OS=" + result + "'",
-               "echo 'export PACKAGES_URL=" + packages_url + "'",
-             ],
-           },
-         ] +
          (if (event == "cron") then [pipeline.publish("pkg latest", "latest")] else []) +
          [pipeline.smoke] +
          [pipeline.smokelog] +

.gitignore

@@ -153,6 +153,7 @@ oam/install_scripts/mcs-workernode.service
 oam/install_scripts/mcs-writeengineserver.service
 oam/install_scripts/mcs-loadbrm.py
 oam/install_scripts/mcs-savebrm.py
+oam/install_scripts/mcssavebrm.py
 oam/install_scripts/mcs-storagemanager.service
 oam/install_scripts/mcs_module_installer.sh
 oam/install_scripts/slave-rep-columnstore.sh
@@ -186,4 +187,3 @@ versioning/BRM/shmkeys.cpp
 obj/
 build/build
 
-