Changes to support direct download of media on Managed Host

docs/user-guide.md

@@ -1059,11 +1059,30 @@ repository types:
 
 #### Cloud Storage bucket
 
-To use a Cloud Storage bucket to stage your installation media, you need the
-`gsutil` tool installed on your control node. The [`gsutil` tool](https://cloud.google.com/storage/docs/gsutil)
-is a Python application that lets you access Cloud Storage from the command
-line. To get the `gsutil` tool, install the [Cloud
-SDK](https://cloud.google.com/sdk/docs).
+If you stage your installation media in a Cloud Storage bucket, the toolkit
+will copy the required files to the software library path location on your
+target server using the Google Cloud CLI `gcloud storage` command. Either
+directly downloading the files on the target server or by transferring them
+through your Ansible Control Node. Consequently, the CLI must be installed on
+your Ansible Control Node or the target server. To install, refer to [Install
+the gcloud CLI](https://cloud.google.com/sdk/docs/install).
+
+When the `--ora-swlib-type` argument is used and set to `GCSDIRECT`, the
+toolkit will directly copy the media from the GCS bucket to the library path
+on the target server. This direct copy requires the gcloud utility to be
+available and the storage bucket accessible from the target server. This is
+often the most performant method for copying media from storage buckets.
+
+If the argument value is `GCSTRANSFER`, the toolkit will instead copy the
+media files through your Ansible Control Node. This method also uses the
+gcloud command, but runs it from your Ansible Control Node. And still
+writes the contents to the software library path on the target server.
+
+When the argument value is `GCS` or is not provided, the toolkit will
+determine which method to use. If the gcloud utility is available on the
+target server and the GCS bucket is accessible from it, the direct method will
+be used. Otherwise, if either condition is not met, the toolkit will revert to
+the transfer method.
 
 #### Cloud Storage FUSE
 
@@ -1550,6 +1569,8 @@ ORA_SWLIB_TYPE
 </pre></p>
 </td>
 <td>GCS<br>
+GCSDIRECT<br>
+GCSTRANSFER<br>
 GCSFUSE<br>
 NFS</td>
 <td>Remote storage type acting as a software library where the required
@@ -1566,7 +1587,7 @@ Example: gs://oracle-software</td>
 <td>GCS bucket where the required base software and patches have been
 downloaded and staged.<br>
 <br>
-Only used when ORA_SWLIB_TYPE=GCS.</td>
+Only used when ORA_SWLIB_TYPE=GCS|GCSDIRECT|GCSTRANSFER.</td>
 </tr>
 <tr>
 <td>Software library path</td>
@@ -1579,7 +1600,7 @@ ORA_SWLIB_PATH
 <td>Path where the required base software and patches have been downloaded and
 staged.<br>
 <br>
-Not used when ORA_SWLIB_TYPE=GCS.</td>
+Not used when ORA_SWLIB_TYPE=GCS|GCSDIRECT|GCSTRANSFER.</td>
 </tr>
 <tr>
 <td>Service account key file</td>
@@ -1994,7 +2015,7 @@ NFS_BACKUP_CONFIG
 Example: nfsv3 </td>
 <td>The NFS version of the export shared is defined with this option. The values accepted are `nfsv3` or `nfsv4`.
 </td>
-</tr> 
+</tr>
 <tr>
 <td>NFS backup mount</td>
 <td><p><pre>

install-oracle.sh

@@ -104,7 +104,7 @@ ORA_SWLIB_BUCKET="${ORA_SWLIB_BUCKET}"
 ORA_SWLIB_BUCKET_PARAM="^.+[^/]"
 
 ORA_SWLIB_TYPE="${ORA_SWLIB_TYPE:-GCS}"
-ORA_SWLIB_TYPE_PARAM="^(\"\"|GCS|GCSFUSE|NFS)$"
+ORA_SWLIB_TYPE_PARAM="^(\"\"|GCS|GCSFUSE|NFS|GCSDIRECT|GCSTRANSFER)$"
 
 ORA_SWLIB_PATH="${ORA_SWLIB_PATH:-/u01/swlib}"
 ORA_SWLIB_PATH_PARAM="^/.*"

roles/swlib/tasks/gcsdirect.yml

@@ -0,0 +1,104 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+# files with alt_name are stored as name for simplicity
+- name: gcsdirect | Copy RDBMS and GI software from GCS to target instance
+  shell: |
+    set -o pipefail
+    {% for f in item.files %}
+    if gcloud storage ls gs://{{ swlib_mount_src }}/{{ f.name }} >/dev/null 2>&1; then
+      f_name="{{ f.name }}"
+    else
+      f_name="{{ f.alt_name | default('x') }}"
+    fi
+    if ! unzip -l "{{ swlib_path }}/{{ f.name }}"
+      then
+        gcloud storage cp gs://{{ swlib_mount_src }}/${f_name} "{{ swlib_path }}/{{ f.name }}"
+    fi
+    {% endfor %}
+  register: shell_result
+  args:
+    executable: /bin/bash
+  with_items:
+    - "{{ rdbms_software }}"
+    - "{{ gi_software }}"
+    - "{{ gi_interim_patches }}"
+  when:
+    - item.version == oracle_ver
+    - patch_only is not defined
+    - not free_edition
+  changed_when: "'Date    Time    Name' not in shell_result.stdout"
+
+- name: gcsdirect | Copy GI patches from GCS to target instance
+  shell: |
+    set -o pipefail
+    if ! unzip -l "{{ swlib_path }}/{{ item.patchfile }}"
+      then
+        gcloud storage cp gs://{{ swlib_mount_src }}/{{ item.patchfile }} "{{ swlib_path }}/{{ item.patchfile }}"
+    fi
+  register: shell_result
+  args:
+    executable: /bin/bash
+  with_items: "{{ gi_patches }}"
+  when: item.release == oracle_rel
+  changed_when: "'Date    Time    Name' not in shell_result.stdout"
+
+# overwrite OPatch zipfile on database host from gcs every time that may be cached currently in {{ swlib_path }}
+# reason to overwrite is that the OPatch zipfile in MOS is named the same, example: p6880880_190000_Linux-x86-64.zip
+# this change downloads OPatch from GCS bucket every time with negligible overhead as OPatch zip file is typically ~120M
+- name: gcsdirect | Copy OPatch update files from GCS to target instance
+  shell: |
+    set -o pipefail
+    gcloud storage cp gs://{{ swlib_mount_src }}/{{ item.patchfile }} "{{ swlib_path }}/{{ item.patchfile }}"
+  register: shell_result
+  args:
+    executable: /bin/bash
+  with_items: "{{ opatch_patches }}"
+  when:
+    - item.release == oracle_ver
+    - oracle_rel != "base"
+  changed_when: "'Date    Time    Name' not in shell_result.stdout"
+
+- name: gcsdirect | Copy RDBMS patches from GCS to target instance
+  shell: |
+    set -o pipefail
+    if ! unzip -l "{{ swlib_path }}/{{ item.patchfile }}"
+      then
+        gcloud storage cp gs://{{ swlib_mount_src }}/{{ item.patchfile }} "{{ swlib_path }}/{{ item.patchfile }}"
+    fi
+  register: shell_result
+  args:
+    executable: /bin/bash
+  with_items: "{{ rdbms_patches }}"
+  when: item.release == oracle_rel
+  changed_when: "'Date    Time    Name' not in shell_result.stdout"
+
+- name: gcsdirect | Copy RPM software from GCS to target instance
+  shell: |
+    set -o pipefail
+    {% for f in item.files %}
+    if ! rpm -qp "{{ swlib_path }}/{{ f.name }}"
+      then
+        gcloud storage cp gs://{{ swlib_mount_src }}/{{ f.name }} "{{ swlib_path }}/{{ f.name }}" && echo "File copied"
+    fi
+    {% endfor %}
+  register: shell_result
+  args:
+    executable: /bin/bash
+  with_items: "{{ rdbms_software }}"
+  when:
+    - item.version == oracle_ver
+    - free_edition
+  changed_when: "'File copied' in shell_result.stdout"

roles/swlib/tasks/gcscopy.yml → roles/swlib/tasks/gcstransfer.yml

@@ -14,19 +14,19 @@
 
 ---
 # files with alt_name are stored as name for simplicity
-- name: gcscopy | Copy RDBMS and GI software from GCS to target instance
+- name: gcstransfer | Transfer RDBMS and GI software from GCS to target instance
   shell: |
     set -o pipefail
     {% for f in item.files %}
-    if gsutil ls gs://{{ swlib_mount_src }}/{{ f.name }} >/dev/null 2>&1; then
+    if gcloud storage ls gs://{{ swlib_mount_src }}/{{ f.name }} >/dev/null 2>&1; then
       f_name="{{ f.name }}"
     else
       f_name="{{ f.alt_name | default('x') }}"
     fi
     if ! ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} {{ ansible_ssh_user }}@{{ ansible_ssh_host }} \
       "unzip -l {{ swlib_path }}/{{ f.name }}"
       then
-        gsutil cat gs://"{{ swlib_mount_src }}"/"${f_name}" | ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} \
+        gcloud storage cat gs://"{{ swlib_mount_src }}"/"${f_name}" | ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} \
           {{ ansible_ssh_user }}@{{ ansible_ssh_host }} "cat > {{ swlib_path }}/{{ f.name }}"
     fi
     {% endfor %}
@@ -44,79 +44,73 @@
   changed_when: "'Date    Time    Name' not in shell_result.stdout"
   delegate_to: 127.0.0.1
 
-- name: gcscopy | Copy GI patches from GCS to target instance
+- name: gcstransfer | Transfer GI patches from GCS to target instance
   shell: |
     set -o pipefail
     if ! ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} {{ ansible_ssh_user }}@{{ ansible_ssh_host }} \
       "unzip -l {{ swlib_path }}/{{ item.patchfile }}"
       then
-        gsutil cat gs://{{ swlib_mount_src }}/{{ item.patchfile }} | ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} \
+        gcloud storage cat gs://{{ swlib_mount_src }}/{{ item.patchfile }} | ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} \
           {{ ansible_ssh_user }}@{{ ansible_ssh_host }} "cat > {{ swlib_path }}/{{ item.patchfile }}"
     fi
   register: shell_result
   args:
     executable: /bin/bash
-  with_items:
-    - "{{ gi_patches }}"
-  when:
-    - item.release == oracle_rel
+  with_items: "{{ gi_patches }}"
+  when: item.release == oracle_rel
   changed_when: "'Date    Time    Name' not in shell_result.stdout"
   delegate_to: 127.0.0.1
 
-# overwrite opatch zipfile on database host from gcs every time that may be cached currently in {{ swlib_path }}
-# reason to overwrite is that the opatch zipfile in MOS is named the same, example: p6880880_190000_Linux-x86-64.zip
-# this change downloads OPatch from GCS bucket every time with neglible overhead as opatch zip file is typically ~120M
-- name: gcscopy | Copy GI OPatch update files from GCS to target instance
+# overwrite OPatch zipfile on database host from gcs every time that may be cached currently in {{ swlib_path }}
+# reason to overwrite is that the OPatch zipfile in MOS is named the same, example: p6880880_190000_Linux-x86-64.zip
+# this change downloads OPatch from GCS bucket every time with negligible overhead as OPatch zip file is typically ~120M
+- name: gcstransfer | Transfer OPatch update files from GCS to target instance
   shell: |
     set -o pipefail
-    gsutil cat gs://{{ swlib_mount_src }}/{{ item.patchfile }} | ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} \
+    gcloud storage cat gs://{{ swlib_mount_src }}/{{ item.patchfile }} | ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} \
       {{ ansible_ssh_user }}@{{ ansible_ssh_host }} "cat > {{ swlib_path }}/{{ item.patchfile }}"
   register: shell_result
   args:
     executable: /bin/bash
-  with_items:
-    - "{{ opatch_patches }}"
+  with_items: "{{ opatch_patches }}"
   when:
     - item.release == oracle_ver
     - oracle_rel != "base"
   changed_when: "'Date    Time    Name' not in shell_result.stdout"
   delegate_to: 127.0.0.1
 
-- name: gcscopy | Copy RDBMS patches from GCS to target instance
+- name: gcstransfer | Transfer RDBMS patches from GCS to target instance
   shell: |
     set -o pipefail
     if ! ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} {{ ansible_ssh_user }}@{{ ansible_ssh_host }} \
       "unzip -l {{ swlib_path }}/{{ item.patchfile }}"
       then
-        gsutil cat gs://{{ swlib_mount_src }}/{{ item.patchfile }} | ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} \
+        gcloud storage cat gs://{{ swlib_mount_src }}/{{ item.patchfile }} | ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} \
           {{ ansible_ssh_user }}@{{ ansible_ssh_host }} "cat > {{ swlib_path }}/{{ item.patchfile }}"
     fi
   register: shell_result
   args:
     executable: /bin/bash
-  with_items:
-    - "{{ rdbms_patches }}"
-  when:
-    - item.release == oracle_rel
+  with_items: "{{ rdbms_patches }}"
+  when: item.release == oracle_rel
   changed_when: "'Date    Time    Name' not in shell_result.stdout"
   delegate_to: 127.0.0.1
 
-- name: gcscopy | Copy RPM software from GCS to target instance
+- name: gcstransfer | Transfer RPM software from GCS to target instance
   shell: |
     set -o pipefail
     {% for f in item.files %}
     if ! ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} {{ ansible_ssh_user }}@{{ ansible_ssh_host }} \
       "rpm -qp {{ swlib_path }}/{{ f.name }}"
       then
-        gsutil cat gs://"{{ swlib_mount_src }}"/"{{ f.name }}" | ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} \
+        gcloud storage cat gs://"{{ swlib_mount_src }}"/"{{ f.name }}" | ssh -i {{ ansible_ssh_private_key_file }} {{ ansible_ssh_extra_args }} \
           {{ ansible_ssh_user }}@{{ ansible_ssh_host }} "cat > {{ swlib_path }}/{{ f.name }}" && echo "File copied"
     fi
     {% endfor %}
   register: shell_result
   args:
     executable: /bin/bash
-  with_items:
-    - "{{ rdbms_software }}"
+  with_items: "{{ rdbms_software }}"
   when:
     - item.version == oracle_ver
     - free_edition

roles/swlib/tasks/main.yml

@@ -32,6 +32,18 @@
       include_tasks: gcsfuse.yml
       when: swlib_mount_type == "gcsfuse"
 
-- name: swlib | gcscopy
-  include_tasks: gcscopy.yml
-  when: swlib_mount_type == "gcs"
+- name: swlib | Check if gcloud is available on the Managed Host and the storage bucket is accessible
+  shell: |
+    gcloud storage ls gs://{{ swlib_mount_src }} >/dev/null 2>&1
+    echo $?
+  when: swlib_mount_type == "gcs" or swlib_mount_type == "gcsdirect"
+  register: gcloud_found
+  changed_when: false
+
+- name: swlib | Copy files from GCS directly on the Managed Host instance
+  include_tasks: gcsdirect.yml
+  when: (swlib_mount_type == "gcsdirect" or swlib_mount_type == "gcs") and (gcloud_found is defined and gcloud_found.stdout == "0")
+
+- name: swlib | Use the Ansible Control Node to transfer files from GCS to the Managed Host instance
+  include_tasks: gcstransfer.yml
+  when: swlib_mount_type == "gcstransfer" or gcloud_found is not defined or (gcloud_found is defined and gcloud_found.stdout != "0")